diff --git a/metadrive/component/map/base_map.py b/metadrive/component/map/base_map.py index be5ba827d..f06a0a6df 100644 --- a/metadrive/component/map/base_map.py +++ b/metadrive/component/map/base_map.py @@ -187,7 +187,8 @@ def get_semantic_map( pixels_per_meter=8, color_setting=MapTerrainSemanticColor, line_sample_interval=2, - polyline_thickness=1, + yellow_line_thickness=1, + white_line_thickness=1, layer=("lane_line", "lane") ): """ @@ -212,7 +213,7 @@ def get_semantic_map( points_to_skip = math.floor(PGDrivableAreaProperty.STRIPE_LENGTH * 2 / line_sample_interval) for obj in all_lanes.values(): if MetaDriveType.is_lane(obj["type"]) and "lane" in layer: - polygons.append((obj["polygon"], MapTerrainSemanticColor.get_color(obj["type"]))) + polygons.append((obj["polygon"], color_setting.get_color(obj["type"]))) elif "lane_line" in layer and (MetaDriveType.is_road_line(obj["type"]) or MetaDriveType.is_road_boundary_line(obj["type"])): if MetaDriveType.is_broken_line(obj["type"]): @@ -220,15 +221,15 @@ def get_semantic_map( if index + points_to_skip < len(obj["polyline"]): polylines.append( ( - [obj["polyline"][index], obj["polyline"][index + points_to_skip]], - MapTerrainSemanticColor.get_color(obj["type"]) + [obj["polyline"][index], + obj["polyline"][index + points_to_skip]], color_setting.get_color(obj["type"]) ) ) else: - polylines.append((obj["polyline"], MapTerrainSemanticColor.get_color(obj["type"]))) + polylines.append((obj["polyline"], color_setting.get_color(obj["type"]))) size = int(size * pixels_per_meter) - mask = np.zeros([size, size, 1], dtype=np.float32) + mask = np.zeros([size, size, 1], dtype=np.uint8) mask[..., 0] = color_setting.get_color(MetaDriveType.GROUND) # create an example bounding box polygon # for idx in range(len(polygons)): @@ -247,8 +248,8 @@ def get_semantic_map( int((p[1] - center_p[1]) * pixels_per_meter) + size / 2 ] for p in line ] - thickness = polyline_thickness * 2 if color == MapTerrainSemanticColor.YELLOW else polyline_thickness - thickness = min(thickness, 2) # clip + thickness = yellow_line_thickness if color == MapTerrainSemanticColor.YELLOW else white_line_thickness + # thickness = min(thickness, 2) # clip cv2.polylines(mask, np.array([points]).astype(np.int32), False, color, thickness) if "crosswalk" in layer: @@ -269,9 +270,8 @@ def get_semantic_map( ) # 0-2pi angle = np.arctan2(*dir) / np.pi * 180 + 180 - # normalize to 0.4-0.714 - angle = angle / 1000 + MapTerrainSemanticColor.get_color(MetaDriveType.CROSSWALK) - cv2.fillPoly(mask, np.array([points]).astype(np.int32), color=angle.tolist()) + angle = int(angle / 2) + color_setting.get_color(MetaDriveType.CROSSWALK) + cv2.fillPoly(mask, np.array([points]).astype(np.int32), color=angle) # self._semantic_map = mask # return self._semantic_map diff --git a/metadrive/constants.py b/metadrive/constants.py index 31e427b69..cf97b463b 100644 --- a/metadrive/constants.py +++ b/metadrive/constants.py @@ -405,8 +405,8 @@ class MapTerrainSemanticColor: Do not modify this as it is for terrain generation. If you want your own palette, just add a new one or modify class lMapSemanticColor """ - YELLOW = 0.1 - WHITE = 0.3 + YELLOW = 10 + WHITE = 30 @staticmethod def get_color(type): @@ -426,10 +426,10 @@ def get_color(type): return MapTerrainSemanticColor.YELLOW elif MetaDriveType.is_lane(type): # return (0, 1, 0, 0) - return 0.2 + return 20 elif type == MetaDriveType.GROUND: # return (0, 0, 1, 0) - return 0.0 + return 00 elif MetaDriveType.is_white_line(type) or MetaDriveType.is_road_boundary_line(type): # return (0, 0, 0, 1) return MapTerrainSemanticColor.WHITE @@ -437,7 +437,7 @@ def get_color(type): # The range of crosswalk value is 0.4 <= value < 0.76, # so people can save the angle (degree) of the crosswalk in attribute map # the value * 10 = angle of crosswalk. It is a trick for saving memory. - return 0.4 # this value can be overwritten latter + return 40 # this value can be overwritten latter else: raise ValueError("Unsupported type: {}".format(type)) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -487,8 +487,7 @@ class TerrainProperty: """ Define some constants/properties for the map and terrain """ - map_region_size = 512 - terrain_size = 2048 + map_region_size = 2048 @classmethod def get_semantic_map_pixel_per_meter(cls): @@ -497,8 +496,8 @@ def get_semantic_map_pixel_per_meter(cls): Returns: a constant """ - assert cls.terrain_size <= 2048, "Terrain size should be fixed to 2048" - return 22 if cls.map_region_size <= 1024 else 11 + # assert cls.terrain_size <= 2048, "Terrain size should be fixed to 2048" + return 22 if cls.map_region_size != 4096 else 11 @classmethod def point_in_map(cls, point): diff --git a/metadrive/engine/core/terrain.py b/metadrive/engine/core/terrain.py index dde5aef33..66a0d2bdb 100644 --- a/metadrive/engine/core/terrain.py +++ b/metadrive/engine/core/terrain.py @@ -1,17 +1,12 @@ # import numpyf import math - import os import pathlib -import sys - +import time # # from abc import ABC -import numpy - -from metadrive.constants import TerrainProperty, CameraTagStateKey import cv2 import numpy as np from panda3d.bullet import BulletRigidBodyNode, BulletPlaneShape @@ -20,8 +15,9 @@ from panda3d.core import Vec3, ShaderTerrainMesh, Texture, TextureStage, Shader, Filename from metadrive.base_class.base_object import BaseObject -from metadrive.constants import CamMask, Semantics +from metadrive.constants import CamMask, Semantics, MapTerrainSemanticColor from metadrive.constants import MetaDriveType, CollisionGroup +from metadrive.constants import TerrainProperty, CameraTagStateKey from metadrive.engine.asset_loader import AssetLoader from metadrive.engine.logger import get_logger from metadrive.third_party.diamond_square import diamond_square @@ -53,7 +49,7 @@ def __init__(self, show_terrain, engine): self.mesh_collision_terrain = None # a 3d mesh, Not available yet! # visualization mesh feature - self._terrain_size = TerrainProperty.terrain_size # [m] + self._terrain_size = TerrainProperty.map_region_size # [m] self._height_scale = engine.global_config["height_scale"] # [m] self._drivable_area_extension = engine.global_config["drivable_area_extension"] # [m] road marin # it should include the whole map. Otherwise, road will have no texture! @@ -83,7 +79,8 @@ def __init__(self, show_terrain, engine): # prepare semantic texture semantic_size = self._semantic_map_size * self._semantic_map_pixel_per_meter self.semantic_tex = Texture() - self.semantic_tex.setup2dTexture(semantic_size, semantic_size, Texture.TFloat, Texture.F_red) + self.semantic_tex.setup2dTexture(semantic_size, semantic_size, Texture.TUnsignedByte, Texture.F_red) + # prepare height field texture self.heightfield_tex = Texture() self.heightfield_tex.setup2dTexture(*self.heightfield_img.shape[:2], Texture.TShort, Texture.FLuminance) @@ -258,18 +255,23 @@ def _set_terrain_shader(self, engine, attribute_tex): # self._mesh_terrain.set_shader_input("side_tex", self.side_tex) # self._mesh_terrain.set_shader_input("side_normal", self.side_normal) - # road + # rock self._mesh_terrain.set_shader_input("rock_tex", self.rock_tex) self._mesh_terrain.set_shader_input("rock_normal", self.rock_normal) self._mesh_terrain.set_shader_input("rock_rough", self.rock_rough) self._mesh_terrain.set_shader_input("rock_tex_ratio", self.rock_tex_ratio) + # rock 2 + self._mesh_terrain.set_shader_input("rock_tex_2", self.rock_tex_2) + self._mesh_terrain.set_shader_input("rock_normal_2", self.rock_normal_2) + self._mesh_terrain.set_shader_input("rock_rough_2", self.rock_rough_2) + self._mesh_terrain.set_shader_input("rock_tex_ratio_2", self.rock_tex_ratio_2) + + # road self._mesh_terrain.set_shader_input("road_tex", self.road_texture) - self._mesh_terrain.set_shader_input("yellow_tex", self.yellow_lane_line) - self._mesh_terrain.set_shader_input("white_tex", self.white_lane_line) + self._mesh_terrain.set_shader_input("road_tex_ratio", self.road_tex_ratio) self._mesh_terrain.set_shader_input("road_normal", self.road_texture_normal) self._mesh_terrain.set_shader_input("road_rough", self.road_texture_rough) - self._mesh_terrain.set_shader_input("elevation_texture_ratio", self._elevation_texture_ratio) # crosswalk self._mesh_terrain.set_shader_input("crosswalk_tex", self.crosswalk_tex) @@ -410,29 +412,18 @@ def _load_mesh_terrain_textures(self, engine, anisotropic_degree=16, filter_type self.ts_normal.setMode(TextureStage.M_normal) # grass - # if engine.use_render_pipeline: - # # grass - # self.grass_tex = self.loader.loadTexture( - # AssetLoader.file_path("textures", "grass2", "grass_path_2_diff_1k.png") - # ) - # self.grass_normal = self.loader.loadTexture( - # AssetLoader.file_path("textures", "grass2", "grass_path_2_nor_gl_1k.png") - # ) - # self.grass_rough = self.loader.loadTexture( - # AssetLoader.file_path("textures", "grass2", "grass_path_2_rough_1k.png") - # ) - # self.grass_tex_ratio = 128.0 - # else: self.grass_tex = self.loader.loadTexture( AssetLoader.file_path("textures", "grass1", "GroundGrassGreen002_COL_1K.jpg") ) self.grass_normal = self.loader.loadTexture( AssetLoader.file_path("textures", "grass1", "GroundGrassGreen002_NRM_1K.jpg") ) - self.grass_rough = self.loader.loadTexture( - AssetLoader.file_path("textures", "grass2", "grass_path_2_rough_1k.png") - ) - self.grass_tex_ratio = 64 + + white = PNMImage(256, 256, 4) + white.fill(1., 1., 1.) + self.grass_rough = Texture("grass rough") + self.grass_rough.load(white) + self.grass_tex_ratio = 64 * self._terrain_size / 512 v_wrap = Texture.WMRepeat u_warp = Texture.WMMirror @@ -454,7 +445,7 @@ def _load_mesh_terrain_textures(self, engine, anisotropic_degree=16, filter_type self.rock_rough = self.loader.loadTexture( AssetLoader.file_path("textures", "rock", "brown_mud_leaves_01_rough_1k.png") ) - self.rock_tex_ratio = 128 + self.rock_tex_ratio = 512 * self._terrain_size / 2048 v_wrap = Texture.WMRepeat u_warp = Texture.WMMirror @@ -466,6 +457,27 @@ def _load_mesh_terrain_textures(self, engine, anisotropic_degree=16, filter_type tex.setMagfilter(filter_type) tex.setAnisotropicDegree(1) + # rock 2 + self.rock_tex_2 = self.loader.loadTexture( + AssetLoader.file_path("textures", "grass2", "grass_path_2_diff_1k.png") + ) + self.rock_normal_2 = self.loader.loadTexture( + AssetLoader.file_path("textures", "grass2", "grass_path_2_nor_gl_1k.png") + ) + self.rock_rough_2 = self.loader.loadTexture( + AssetLoader.file_path("textures", "grass2", "grass_path_2_rough_1k.png") + ) + self.rock_tex_ratio_2 = 256 * self._terrain_size / 2048 + + v_wrap = Texture.WMRepeat + u_warp = Texture.WMMirror + + for tex in [self.rock_tex_2, self.rock_normal_2, self.rock_rough_2]: + tex.set_wrap_u(u_warp) + tex.set_wrap_v(v_wrap) + tex.setMinfilter(filter_type) + tex.setMagfilter(filter_type) + # # sidewalk # self.side_tex = self.loader.loadTexture(AssetLoader.file_path("textures", "sidewalk", "color.png")) # self.side_normal = self.loader.loadTexture(AssetLoader.file_path("textures", "sidewalk", "normal.png")) @@ -481,16 +493,15 @@ def _load_mesh_terrain_textures(self, engine, anisotropic_degree=16, filter_type # tex.setAnisotropicDegree(1) # Road surface - # self.road_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "sci", "new_color.png")) self.road_texture = self.loader.loadTexture(AssetLoader.file_path("textures", "asphalt", "diff_2k.png")) self.road_texture_normal = self.loader.loadTexture( AssetLoader.file_path("textures", "asphalt", "normal_2k.png") ) self.road_texture_rough = self.loader.loadTexture(AssetLoader.file_path("textures", "asphalt", "rough_2k.png")) + self.road_tex_ratio = 128 * self._terrain_size / 2048 v_wrap = Texture.WMRepeat u_warp = Texture.WMMirror filter_type = Texture.FTLinearMipmapLinear - anisotropic_degree = 16 for tex in [self.road_texture_rough, self.road_texture, self.road_texture_normal]: tex.set_wrap_u(u_warp) tex.set_wrap_v(v_wrap) @@ -502,15 +513,15 @@ def _load_mesh_terrain_textures(self, engine, anisotropic_degree=16, filter_type # self.road_texture.setAnisotropicDegree(1) # lane line - white_lane_line = PNMImage(1024, 1024, 4) - white_lane_line.fill(1., 1., 1.) - self.white_lane_line = Texture("white lane line") - self.white_lane_line.load(white_lane_line) - - yellow_lane_line = PNMImage(1024, 1024, 4) - yellow_lane_line.fill(*(255 / 255, 200 / 255, 0 / 255)) - self.yellow_lane_line = Texture("white lane line") - self.yellow_lane_line.load(yellow_lane_line) + # white_lane_line = PNMImage(256, 256, 4) + # white_lane_line.fill(1., 1., 1.) + # self.white_lane_line = Texture("white lane line") + # self.white_lane_line.load(white_lane_line) + # + # yellow_lane_line = PNMImage(256, 256, 4) + # yellow_lane_line.fill(*(255 / 255, 200 / 255, 0 / 255)) + # self.yellow_lane_line = Texture("white lane line") + # self.yellow_lane_line.load(yellow_lane_line) # crosswalk tex = np.frombuffer(self.road_texture.getRamImage().getData(), dtype=np.uint8) @@ -601,14 +612,16 @@ def get_terrain_semantics(self, center_point): center_point, size=self._semantic_map_size, pixels_per_meter=self._semantic_map_pixel_per_meter, - polyline_thickness=int(self._semantic_map_pixel_per_meter / 11), + white_line_thickness=2, + yellow_line_thickness=3, # 1 when map_region_size == 2048, 2 for others layer=layer ) else: logger.warning("Can not find map. Generate a square terrain") size = self._semantic_map_size * self._semantic_map_pixel_per_meter - semantics = np.ones((size, size, 1), dtype=np.float32) * 0.2 + lane_color = MapTerrainSemanticColor.get_color(MetaDriveType.LANE_SURFACE_STREET) + semantics = np.ones((size, size, 1), dtype=np.uint8) * lane_color # use lane color return semantics @staticmethod diff --git a/metadrive/envs/base_env.py b/metadrive/envs/base_env.py index 86709b839..2606dba3f 100644 --- a/metadrive/envs/base_env.py +++ b/metadrive/envs/base_env.py @@ -2,7 +2,7 @@ import time from collections import defaultdict from typing import Union, Dict, AnyStr, Optional, Tuple, Callable -from metadrive.constants import DEFAULT_SENSOR_HPR, DEFAULT_SENSOR_OFFSET + import gymnasium as gym import numpy as np from panda3d.core import PNMImage @@ -12,6 +12,7 @@ from metadrive.component.sensors.dashboard import DashBoard from metadrive.component.sensors.distance_detector import LaneLineDetector, SideDetector from metadrive.component.sensors.lidar import Lidar +from metadrive.constants import DEFAULT_SENSOR_HPR, DEFAULT_SENSOR_OFFSET from metadrive.constants import RENDER_MODE_NONE, DEFAULT_AGENT from metadrive.constants import RENDER_MODE_ONSCREEN, RENDER_MODE_OFFSCREEN from metadrive.constants import TerminationState, TerrainProperty @@ -217,7 +218,7 @@ # ===== Terrain ===== # The size of the square map region, which is centered at [0, 0]. The map objects outside it are culled. - map_region_size=1024, + map_region_size=2048, # Whether to remove lanes outside the map region. If True, lane localization only applies to map region cull_lanes_outside_map=False, # Road will have a flat marin whose width is determined by this value, unit: [m] @@ -331,7 +332,7 @@ def _post_process_config(self, config): # Adjust terrain n = config["map_region_size"] - assert (n & (n - 1)) == 0 and 0 < n <= 2048, "map_region_size should be pow of 2 and < 2048." + assert (n & (n - 1)) == 0 and 512 <= n <= 4096, "map_region_size should be pow of 2 and < 2048." TerrainProperty.map_region_size = config["map_region_size"] # Multi-Thread @@ -440,7 +441,7 @@ def step(self, actions: Union[Union[np.ndarray, list], Dict[AnyStr, Union[list, return self._get_step_return(actions, engine_info=engine_info) # collect observation, reward, termination def _preprocess_actions(self, actions: Union[np.ndarray, Dict[AnyStr, np.ndarray], int]) \ - -> Union[np.ndarray, Dict[AnyStr, np.ndarray], int]: + -> Union[np.ndarray, Dict[AnyStr, np.ndarray], int]: if not self.is_multi_agent: actions = {v_id: actions for v_id in self.agents.keys()} else: @@ -637,7 +638,7 @@ def _get_step_return(self, actions, engine_info): if not self.is_multi_agent: return self._wrap_as_single_agent(obses), self._wrap_as_single_agent(rewards), \ - self._wrap_as_single_agent(terminateds), self._wrap_as_single_agent( + self._wrap_as_single_agent(terminateds), self._wrap_as_single_agent( truncateds), self._wrap_info_as_single_agent(step_infos) else: return obses, rewards, terminateds, truncateds, step_infos diff --git a/metadrive/envs/scenario_env.py b/metadrive/envs/scenario_env.py index a8837d587..52414eb0b 100644 --- a/metadrive/envs/scenario_env.py +++ b/metadrive/envs/scenario_env.py @@ -37,7 +37,7 @@ store_data=True, need_lane_localization=True, no_map=False, - map_region_size=512, + map_region_size=2048, cull_lanes_outside_map=True, # ===== Scenario ===== diff --git a/metadrive/examples/drive_in_real_env.py b/metadrive/examples/drive_in_real_env.py index 330108a51..436f708e9 100755 --- a/metadrive/examples/drive_in_real_env.py +++ b/metadrive/examples/drive_in_real_env.py @@ -3,8 +3,10 @@ This script demonstrates how to use the environment where traffic and road map are loaded from Waymo dataset. """ import argparse -import random +from metadrive.component.sensors.depth_camera import DepthCamera +from metadrive.component.sensors.rgb_camera import RGBCamera +from metadrive.component.sensors.semantic_camera import SemanticCamera from metadrive.constants import HELP_MESSAGE from metadrive.engine.asset_loader import AssetLoader from metadrive.envs.scenario_env import ScenarioEnv @@ -21,24 +23,35 @@ parser.add_argument("--reactive_traffic", action="store_true") parser.add_argument("--top_down", "--topdown", action="store_true") parser.add_argument("--waymo", action="store_true") + parser.add_argument("--add_sensor", action="store_true") args = parser.parse_args() extra_args = dict(film_size=(2000, 2000)) if args.top_down else {} asset_path = AssetLoader.asset_path use_waymo = args.waymo print(HELP_MESSAGE) - try: - env = ScenarioEnv( - { - "manual_control": True, - "sequential_seed": True, - "reactive_traffic": True if args.reactive_traffic else False, - "use_render": True if not args.top_down else False, - "data_directory": AssetLoader.file_path( - asset_path, "waymo" if use_waymo else "nuscenes", unix_style=False - ), - "num_scenarios": 3 if use_waymo else 10 + + cfg = { + "manual_control": True, + "map_region_size": 1024, # use a large number if your map is toooooo big + "sequential_seed": True, + "reactive_traffic": True if args.reactive_traffic else False, + "use_render": True if not args.top_down else False, + "data_directory": AssetLoader.file_path(asset_path, "waymo" if use_waymo else "nuscenes", unix_style=False), + "num_scenarios": 3 if use_waymo else 10 + } + if args.add_sensor: + additional_cfg = { + "interface_panel": ["rgb_camera", "depth_camera", "semantic"], + "sensors": { + "rgb_camera": (DepthCamera, 256, 256), + "depth_camera": (RGBCamera, 256, 256), + "semantic": (SemanticCamera, 256, 256) } - ) + } + cfg.update(additional_cfg) + + try: + env = ScenarioEnv(cfg) o, _ = env.reset() for i in range(1, 100000): diff --git a/metadrive/shaders/terrain.frag.glsl b/metadrive/shaders/terrain.frag.glsl index 133ad37c9..867ada08f 100644 --- a/metadrive/shaders/terrain.frag.glsl +++ b/metadrive/shaders/terrain.frag.glsl @@ -3,7 +3,7 @@ // Number of splits in the PSSM, it must be in line with what is configured in the PSSMCameraRig const int split_count=2; uniform vec3 light_direction; - +#define saturate(v) clamp(v, 0, 1) uniform mat3 p3d_NormalMatrix; uniform struct { @@ -32,26 +32,30 @@ uniform struct { uniform vec3 wspos_camera; // asset -uniform sampler2D yellow_tex; -uniform sampler2D white_tex; uniform sampler2D road_tex; uniform sampler2D road_normal; -uniform sampler2D road_rough; +// uniform sampler2D road_rough; +uniform float road_tex_ratio; uniform sampler2D crosswalk_tex; uniform sampler2D grass_tex; uniform sampler2D grass_normal; -uniform sampler2D grass_rough; +// uniform sampler2D grass_rough; uniform float grass_tex_ratio; uniform sampler2D rock_tex; uniform sampler2D rock_normal; -uniform sampler2D rock_rough; +// uniform sampler2D rock_rough; +uniform float rock_tex_ratio; + +uniform sampler2D rock_tex_2; +uniform sampler2D rock_normal_2; +// uniform sampler2D rock_rough_2; +uniform float rock_tex_ratio_2; uniform sampler2D attribute_tex; // just learned that uniform means the variable won't change in each stage, while in/out is able to do that : ) -uniform float elevation_texture_ratio; uniform float height_scale; uniform sampler2D PSSMShadowAtlas; @@ -76,24 +80,14 @@ vec3 project(mat4 mvp, vec3 p) { } -vec3 get_normal(vec3 diffuse, sampler2D normal_tex, sampler2D rough_tex, float tex_ratio, mat3 tbn){ - vec3 normal = normalize(texture(normal_tex, terrain_uv * tex_ratio).rgb*2.0-1.0); +vec3 get_normal(vec3 diffuse, sampler2D normal_tex, float tex_ratio, mat3 tbn){ + vec3 normal = texture(normal_tex, terrain_uv * tex_ratio).rgb*2.0-1.0; normal = normalize(tbn * normal); return normal; } void main() { - float road_tex_ratio = 128; - float grass_tex_ratio = grass_tex_ratio * 4; - float r_min = (1-1/elevation_texture_ratio)/2; - float r_max = (1-1/elevation_texture_ratio)/2+1/elevation_texture_ratio; - vec4 attri; - if (abs(elevation_texture_ratio - 1) < 0.001) { - attri = texture(attribute_tex, terrain_uv); - } - else { - attri = texture(attribute_tex, terrain_uv*elevation_texture_ratio+0.5); - } + vec4 attri = texture(attribute_tex, terrain_uv); // terrain normal vec3 pixel_size = vec3(1.0, -1.0, 0) / textureSize(ShaderTerrainMesh.heightfield, 0).xxx; @@ -105,61 +99,95 @@ void main() { vec3 binormal = normalize(vec3(0, 1, h_v1 - h_v0)); vec3 terrain_normal = normalize(cross(tangent, binormal)); vec3 normal = normalize(p3d_NormalMatrix * terrain_normal); + vec3 viewDir = normalize(wspos_camera - vtx_pos); + float height = (h_u0 + h_u1 + h_v0 + h_v1) / (4.0 * height_scale); // xxx + float slope = 1.0 - terrain_normal.z; // normal.x *= -1; mat3 tbn = mat3(tangent, binormal, terrain_normal); vec3 shading = vec3(0.0); - // Calculate the shading of each light in the scene - for (int i = 0; i < p3d_LightSource.length(); ++i) { - vec3 diff = p3d_LightSource[i].position.xyz - vtx_pos * p3d_LightSource[i].position.w; - vec3 light_vector = normalize(diff); - vec3 light_shading = clamp(dot(normal, light_vector), 0.0, 1.0) * p3d_LightSource[i].color; - // If PSSM is not used, use the shadowmap from the light - // This is deeply ineficient, it's only to be able to compare the rendered shadows - if (!use_pssm) { - vec4 projected = projecteds[i]; - // Apply a bias to remove some of the self-shadow acne - projected.z -= fixed_bias * 0.01 * projected.w; - light_shading *= textureProj(p3d_LightSource[i].shadowMap, projected); - } - shading += light_shading; - } - // get the color and terrain normal in world space - vec3 diffuse; + vec3 diffuse = vec3(0.0, 0.0, 0.0); vec3 tex_normal_world; - if ((attri.r > 0.01) && (terrain_uv.x>=r_min) && (terrain_uv.y >= r_min) && (terrain_uv.x<=r_max) && (terrain_uv.y<=r_max)){ - float value = attri.r; // Assuming it's a red channel texture - if (value < 0.11) { + // float roughnessValue; + if (attri.r > 0.01){ + float value = attri.r * 255; // Assuming it's a red channel texture + if (value < 14) { // yellow - diffuse=texture(yellow_tex, terrain_uv * road_tex_ratio).rgb; - } else if (value < 0.21) { + diffuse=vec3(1.0, 0.78, 0.0); + } else if (value < 23) { // road diffuse = texture(road_tex, terrain_uv * road_tex_ratio).rgb; - } else if (value < 0.31) { + } else if (value < 31) { // white - diffuse = texture(white_tex, terrain_uv * road_tex_ratio).rgb; - } else if (value > 0.3999 || value < 0.760001) { + diffuse = vec3(1.0, 1.0, 1.0); + } else if (value > 39 || value < 221) { // crosswalk - float theta=(value-0.39999) * 1000/180 * 3.1415926535; + float theta=(value-40) * 2/180.0*3.1415926535; vec2 new_terrain_uv = vec2(cos(theta)*terrain_uv.x - sin(theta)*terrain_uv.y, sin(theta)*terrain_uv.x+cos(theta)*terrain_uv.y); diffuse = texture(crosswalk_tex, new_terrain_uv * road_tex_ratio).rgb; - } else{ - // Semantics for value 4 - diffuse = texture(white_tex, terrain_uv * road_tex_ratio).rgb; } - tex_normal_world = get_normal(diffuse, road_normal, road_rough, road_tex_ratio, tbn); + tex_normal_world = get_normal(diffuse, road_normal, road_tex_ratio, tbn); + // roughnessValue = texture(road_rough, terrain_uv * road_tex_ratio).r; } else{ + // texture splatting + float grass = 0.0; + float rock = 0.0; + float rock_2 = 0.0; - // texture splatting, mixing ratio can be determined via rgba, no grass here - diffuse = texture(grass_tex, terrain_uv * grass_tex_ratio).rgb; - tex_normal_world = get_normal(diffuse, grass_normal, grass_rough, grass_tex_ratio, tbn); + { // rock_2 + rock_2 = saturate(0.8 * (height-0.07)); + rock_2 *= saturate(pow(saturate(1.0 - slope), 2.0)) * 2.0; + + rock_2 = saturate(rock_2); + } + + { // Rock + rock = saturate((pow(slope, 1.2) * 15)); + } + + { // Grass + grass = saturate(1.0 - saturate(rock + rock_2)); + } + + diffuse = diffuse + texture(grass_tex, terrain_uv * grass_tex_ratio).rgb * grass; + diffuse = diffuse + texture(rock_tex, terrain_uv * rock_tex_ratio).rgb * rock; + diffuse = diffuse + texture(rock_tex_2, terrain_uv * rock_tex_ratio_2).rgb * rock_2; + + tex_normal_world = tex_normal_world + (texture(grass_normal, terrain_uv * grass_tex_ratio).rgb*2.0-1.0) * grass; + tex_normal_world = tex_normal_world + (texture(rock_normal, terrain_uv * rock_tex_ratio).rgb*2.0-1.0) * rock; + tex_normal_world = tex_normal_world + (texture(rock_normal_2, terrain_uv * rock_tex_ratio_2).rgb*2.0-1.0) * rock_2; + tex_normal_world = normalize(tbn * tex_normal_world); + + //roughnessValue = roughnessValue + texture(grass_rough, terrain_uv * grass_tex_ratio).r * grass; + //roughnessValue = roughnessValue + texture(rock_rough, terrain_uv * rock_tex_ratio).r * rock; + //roughnessValue = roughnessValue + texture(rock_rough_2, terrain_uv * rock_tex_ratio_2).r * rock_2; + //roughnessValue = saturate(roughnessValue); } // vec3 terrain_normal_view = normalize(tex_normal_world); + // Calculate the shading of each light in the scene + for (int i = 0; i < p3d_LightSource.length(); ++i) { + vec3 diff = p3d_LightSource[i].position.xyz - vtx_pos * p3d_LightSource[i].position.w; + vec3 light_vector = normalize(diff); + vec3 light_shading = clamp(dot(normal, light_vector), 0.0, 1.0) * p3d_LightSource[i].color; + + // Specular (Blinn-Phong example) + // vec3 halfDir = normalize(light_vector + viewDir); + // float NdotH = max(dot(tex_normal_world, halfDir), 0.0); + // float exponent = 2.0 + (1.0 - roughnessValue) * 256.0; + // float spec = pow(NdotH, exponent); + // float specStrength = 0.4; + // vec3 specColor = p3d_LightSource[i].color * spec * specStrength; + // light_shading += specColor; + + + shading += light_shading; + } + // static shadow vec3 light_dir = normalize(light_direction); shading *= max(0.0, dot(tex_normal_world, light_dir)); diff --git a/metadrive/shaders/terrain_effect.yaml b/metadrive/shaders/terrain_effect.yaml index 5410b4c7e..3996b2726 100644 --- a/metadrive/shaders/terrain_effect.yaml +++ b/metadrive/shaders/terrain_effect.yaml @@ -92,15 +92,14 @@ fragment: uniform sampler2D attribute_tex; uniform float grass_tex_ratio; uniform float rock_tex_ratio; + uniform float road_tex_ratio; // just learned that uniform means the variable won't change in each stage, while in/out is able to do that : ) - uniform float elevation_texture_ratio; uniform float height_scale; material: | - float road_tex_ratio = 128; - float grass_tex_ratio = grass_tex_ratio * 4; + float elevation_texture_ratio = 1; const float r_min = (1-1/elevation_texture_ratio)/2; const float r_max = (1-1/elevation_texture_ratio)/2+1/elevation_texture_ratio; @@ -124,20 +123,20 @@ fragment: mat3 tbn = mat3(tangent, binormal, normal); if ((attri.r > 0.01) && terrain_uv.x>=r_min && terrain_uv.y >= r_min && terrain_uv.x<=r_max && terrain_uv.y<=r_max){ - float value = attri.r; // Assuming it's a red channel texture + float value = attri.r * 255; // Assuming it's a red channel texture vec3 diffuse; - if (value < 0.11) { + if (value < 14) { // Semantics for value 1 diffuse=texture(yellow_tex, terrain_uv * road_tex_ratio).rgb; - } else if (value < 0.21) { + } else if (value < 23) { // Semantics for value 2 diffuse = texture(road_tex, terrain_uv * road_tex_ratio).rgb; - } else if (value < 0.31) { + } else if (value < 31) { // Semantics for value 4 diffuse = texture(white_tex, terrain_uv * road_tex_ratio).rgb; - } else if (value > 0.3999 || value < 0.760001) { + } else if (value > 39|| value < 221) { // crosswalk - float theta=(value-0.39999) * 1000/180 * 3.1415926535; + float theta=(value-40) * 2/180.0*3.1415926535; vec2 new_terrain_uv = vec2(cos(theta)*terrain_uv.x - sin(theta)*terrain_uv.y, sin(theta)*terrain_uv.x+cos(theta)*terrain_uv.y); diffuse = texture(crosswalk_tex, new_terrain_uv * road_tex_ratio).rgb; } else { diff --git a/metadrive/shaders/terrain_semantics.frag.glsl b/metadrive/shaders/terrain_semantics.frag.glsl index ca9b8663b..ff6f1efe2 100644 --- a/metadrive/shaders/terrain_semantics.frag.glsl +++ b/metadrive/shaders/terrain_semantics.frag.glsl @@ -3,7 +3,7 @@ // Number of splits in the PSSM, it must be in line with what is configured in the PSSMCameraRig const int split_count=2; uniform vec3 light_direction; - +#define saturate(v) clamp(v, 0, 1) uniform mat3 p3d_NormalMatrix; uniform struct { @@ -40,7 +40,6 @@ uniform vec3 ground_semantics; uniform sampler2D attribute_tex; // just learned that uniform means the variable won't change in each stage, while in/out is able to do that : ) -uniform float elevation_texture_ratio; uniform float height_scale; uniform sampler2D PSSMShadowAtlas; @@ -60,6 +59,7 @@ out vec4 color; void main() { + float elevation_texture_ratio = 1.0; float r_min = (1-1/elevation_texture_ratio)/2; float r_max = (1-1/elevation_texture_ratio)/2+1/elevation_texture_ratio; vec4 attri; @@ -73,21 +73,21 @@ void main() { // get the color and terrain normal in world space vec3 diffuse; if ((attri.r > 0.01) && (terrain_uv.x>=r_min) && (terrain_uv.y >= r_min) && (terrain_uv.x<=r_max) && (terrain_uv.y<=r_max)){ - float value = attri.r;// Assuming it's a red channel texture - if (value < 0.11) { + float value = attri.r * 255;// Assuming it's a red channel texture + if (value < 14) { // yellow diffuse=lane_line_semantics; - } else if (value < 0.21) { + } else if (value < 23) { // road diffuse = road_semantics; - } else if (value < 0.31) { + } else if (value < 31) { // white diffuse = lane_line_semantics; - } else if (value > 0.3999 || value < 0.760001) { + } else if (value > 39 || value < 221) { // crosswalk - float theta=(value-0.39999) * 1000/180 * 3.1415926535; - vec2 new_terrain_uv = vec2(cos(theta)*terrain_uv.x - sin(theta)*terrain_uv.y, sin(theta)*terrain_uv.x+cos(theta)*terrain_uv.y); - diffuse = crosswalk_semantics; + float theta=(value-40) * 2/180.0*3.1415926535; + vec2 new_terrain_uv = vec2(cos(theta)*terrain_uv.x - sin(theta)*terrain_uv.y, sin(theta)*terrain_uv.x+cos(theta)*terrain_uv.y); + diffuse = crosswalk_semantics; } else { // Semantics for value 4 diffuse = road_semantics; diff --git a/metadrive/tests/vis_env/vis_metadrive_env.py b/metadrive/tests/vis_env/vis_metadrive_env.py index 100a7dfe3..7c28a8bcb 100644 --- a/metadrive/tests/vis_env/vis_metadrive_env.py +++ b/metadrive/tests/vis_env/vis_metadrive_env.py @@ -35,7 +35,6 @@ "start_seed": 1010, # "agent_policy": ExpertPolicy, "random_traffic": False, - "map_region_size": 1024, # "height_scale": 100, # "random_lane_width": True, "driving_reward": 1.0, diff --git a/metadrive/tests/vis_functionality/vis_point_cloud.py b/metadrive/tests/vis_functionality/vis_point_cloud.py index d9a3efdad..3b33bd411 100644 --- a/metadrive/tests/vis_functionality/vis_point_cloud.py +++ b/metadrive/tests/vis_functionality/vis_point_cloud.py @@ -10,13 +10,15 @@ "traffic_density": 0., "accident_prob": 1., "start_seed": 4, - "map": "SSSSS", + "map": "CSCSCCCCCC", "manual_control": True, # "use_render": True, "image_observation": True, + "map_region_size": 2048, # "norm_pixel": True, "use_render": True, - "debug": False, + "use_mesh_terrain": True, + "debug": True, "interface_panel": ["point_cloud"], "sensors": dict(point_cloud=(PointCloudLidar, 200, 64, True)), # 64 channel lidar "vehicle_config": dict(image_source="point_cloud"), @@ -31,6 +33,7 @@ env.reset() drawer = env.engine.make_line_drawer() cam = env.engine.get_sensor("point_cloud").cam + env.engine.accept("9", env.engine.terrain.reload_terrain_shader) for i in range(1, 100000): o, r, tm, tc, info = env.step([0, 1]) diff --git a/metadrive/tests/vis_functionality/vis_semantic_cam.py b/metadrive/tests/vis_functionality/vis_semantic_cam.py index d00a09a1f..c22075b70 100644 --- a/metadrive/tests/vis_functionality/vis_semantic_cam.py +++ b/metadrive/tests/vis_functionality/vis_semantic_cam.py @@ -24,7 +24,6 @@ def get_image(env): "num_scenarios": 10, "debug": True, "debug_static_world": True, - "map_region_size": 256, "norm_pixel": True, "show_interface": True, "show_sidewalk": True,