From c75f2718094a098274b44d31b5a40c98ae7e68ee Mon Sep 17 00:00:00 2001 From: Romain Guesdon <r.guesdon@hotmail.fr> Date: Thu, 1 Sep 2022 18:38:12 +0200 Subject: [PATCH] Add hand targets --- global_script.py | 79 ++++++++-------- scripts/camera_proj.py | 2 +- scripts/human.py | 28 +++++- scripts/random_pose.py | 200 ++++++++++++----------------------------- scripts/utils.py | 4 +- 5 files changed, 124 insertions(+), 189 deletions(-) diff --git a/global_script.py b/global_script.py index 9bc51ca..7fd5311 100644 --- a/global_script.py +++ b/global_script.py @@ -27,7 +27,7 @@ importlib.reload(utils) importlib.reload(camera_proj) importlib.reload(human) -from scripts.human import Human, HumanLoader, set_shrinkwraps +from scripts.human import HumanLoader def abs_path(rel_path): @@ -39,6 +39,8 @@ random.seed() C = bpy.context D = bpy.data +CONTINUE = False + # Clean scene try: bpy.ops.object.mode_set(mode='OBJECT') @@ -71,11 +73,11 @@ cars = [] for src_path, name in { r"car_models\suv_car\car.blend": 'SUV', - r"car_models\red_car\red.blend": 'Red', - r"car_models\pickup_car\pickup.blend": 'PickUp', - r"car_models\family_car\family_car.blend": 'Family', - r"car_models\coupe_car\coupe_car.blend": 'Coupe', - r"car_models\truck\truck_open.blend": 'Truck', + # r"car_models\red_car\red.blend": 'Red', + # r"car_models\pickup_car\pickup.blend": 'PickUp', + # r"car_models\family_car\family_car.blend": 'Family', + # r"car_models\coupe_car\coupe_car.blend": 'Coupe', + # r"car_models\truck\truck_open.blend": 'Truck', }.items(): with D.libraries.load(abs_path(src_path)) as (data_from, data_to): @@ -98,7 +100,6 @@ C.scene.render.engine = 'BLENDER_EEVEE' camera_data = D.cameras.new(name='Camera') camera_data.type = 'PERSP' camera_data.lens_unit = 'FOV' -# camera_data.angle = utils.r(68) camera_data.angle = utils.r(68) C.scene.render.resolution_x = 640 @@ -107,8 +108,6 @@ C.scene.render.resolution_y = 480 camera_object = D.objects.new('Camera', camera_data) C.scene.collection.objects.link(camera_object) -# camera_object.location = [-1.1, -0.21, 0.54] -# camera_object.rotation_euler = utils.r([76, 12, -76]) camera_object.location = [-.97, -0.11, 0.68] camera_object.rotation_euler = utils.r([72, 8, -82]) @@ -141,8 +140,6 @@ image_holder.location = (4, 1.5, 1.3) image_holder.rotation_euler.z = utils.r(-100) image_holder.active_material.shadow_method = 'NONE' -# C.scene.node_tree.nodes["Rotate"].inputs[1].default_value = camera_object.rotation_euler[1] - utils.r(5) - # add light sun_collection = D.collections.new("Sun") C.scene.collection.children.link(sun_collection) @@ -180,29 +177,39 @@ C.scene.sun_pos_properties.year = 2022 C.scene.sun_pos_properties.day_of_year = 182 fp = abs_path(r"output") -if os.path.isdir(fp): - shutil.rmtree(fp) -os.mkdir(fp) - fp_img = os.path.join(fp, 'images') fp_ann_2D = os.path.join(fp, 'annots_2D') fp_ann_3D = os.path.join(fp, 'annots_3D') -os.mkdir(fp_img) -os.mkdir(fp_ann_2D) -os.mkdir(fp_ann_3D) + +info_path = os.path.join(fp, 'infos.json') +scenes_ids = OrderedDict() +if not CONTINUE or not os.path.isfile(info_path): + if os.path.isdir(fp): + shutil.rmtree(fp) + os.mkdir(fp) + + os.mkdir(fp_img) + os.mkdir(fp_ann_2D) + os.mkdir(fp_ann_3D) frame_rate = 25 -nb_scene = 2 -nb_pose = 1 +nb_scene = 1 +nb_pose = 10 human_loader.max_len = min(human_loader.max_len, nb_scene) ratio_conf_man = int(nb_scene / len(human_loader.human_paths)) +if CONTINUE: + with open(info_path) as f_info: + scene_ids = json.load(f_info, object_pairs_hook=OrderedDict)['id_max_scenes'] -scenes_ids = OrderedDict() + human_loader.human_paths = [hp for hp in human_loader.human_paths if hp not in scene_ids] +man = None for sc in range(nb_scene): - ## Random car + # Random car car = car_picker() - ## Random personne + car_targets = {side: [ch for ch in car.children if f'Target_{side.upper()}' in ch.name] for side in 'lr'} + nb_targets = sum([len(v) for v in car_targets.values()]) + # Random personne if ratio_conf_man < 1: if not sc % 10: human_loader.load_next() @@ -217,7 +224,7 @@ for sc in range(nb_scene): scenes_ids.setdefault(human_path, -1) scenes_ids[human_path] += 1 - ## Random time + # Random time C.scene.sun_pos_properties.north_offset = utils.r(random.randint(-179, 180)) time_day = random.randint(0, 23) if 6 <= time_day <= 21: @@ -234,16 +241,16 @@ for sc in range(nb_scene): bpy.data.worlds["World"].node_tree.nodes["Background.001"].inputs[0].default_value = \ (back_val, back_val, back_val, 1) - ## Random background + # Random background back_img = random.choice(back_imgs[day_night]) - # C.scene.node_tree.nodes["Image"].image = D.images[back_img] image_holder.active_material.node_tree.nodes['Image Texture'].image = D.images[back_img] + image_holder.location.y = 1.5 + random.uniform(-0.3, 0.3) # Camera movement camera_object.rotation_euler = utils.r([72 + random.randint(-2, 2), 8, -82]) C.scene.render.filepath = fp - C.scene.render.image_settings.file_format = 'PNG' # set output format to .png + C.scene.render.image_settings.file_format = 'PNG' C.scene.camera = camera_object P, K, RT = camera_proj.get_3x4_P_matrix_from_blender(camera_object) @@ -265,8 +272,7 @@ for sc in range(nb_scene): man.animation_data_clear() - ## Pour chaque paire, on veut NbTotalFrames / NbPersonnes / NbCar poses - ## Exemple: 150k / 200 / 2 = 1500 pose + # Exemple: 150k / 200 / 2 = 1500 poses with open(os.path.join(fp_ann_2D, f'annotations_{file_root_name}.csv'), 'w') as annot_file_2D, \ open(os.path.join(fp_ann_3D, f'annotations_{file_root_name}.csv'), 'w') as annot_file_3D: bone_lbls = list(man.pose.bones.keys()) @@ -276,10 +282,11 @@ for sc in range(nb_scene): for po in range(nb_pose): C.scene.frame_set(po * frame_rate) - ### On random une pose à chaque fois - random_pose.random_pose_ik(man) + use_targets = nb_pose - po <= (nb_targets // 2) ** 2 + # use_targets = False + human.switch_constraints(man, enable=not use_targets) + random_pose.random_pose_ik(man, targets=car_targets if use_targets else None) - ### On crée une frame bpy.ops.object.mode_set(mode='OBJECT') man.keyframe_insert(data_path="location", index=-1) @@ -294,12 +301,8 @@ for sc in range(nb_scene): bpy.ops.object.mode_set(mode='OBJECT') - ### Random caméra et lumières (Blender proc) - ### On génère les images au bon FPS - ### On sauvegarde les poses 3D + paramètres caméra (extra et intra) - # set output path so render won't get overwritten - C.scene.render.filepath = os.path.join(fp_img, f"{file_root_name}_{po}") + C.scene.render.filepath = os.path.join(fp_img, f"{file_root_name}_{po}" + f'_drive' if use_targets else '') bpy.ops.render.render(write_still=True) # render still annotations_2D = [] @@ -318,7 +321,7 @@ C.scene.frame_end = int(frame_rate * (nb_pose - 0.5)) utils.select_only(man) bpy.ops.object.mode_set(mode='POSE') -with open(os.path.join(fp, 'infos.json'), 'w') as f: +with open(info_path, 'w') as f: json.dump({ 'models': list(scenes_ids), 'id_max_scenes': scenes_ids diff --git a/scripts/camera_proj.py b/scripts/camera_proj.py index 76329f8..9fd0b76 100644 --- a/scripts/camera_proj.py +++ b/scripts/camera_proj.py @@ -9,7 +9,7 @@ from mathutils import Vector # --------------------------------------------------------------- # Build intrinsic camera parameters from Blender camera data -# +# From https://blender.stackexchange.com/a/38210/153600 # See notes on this in # blender.stackexchange.com/questions/15102/what-is-blenders-camera-projection-matrix-model def get_calibration_matrix_K_from_blender(camd): diff --git a/scripts/human.py b/scripts/human.py index 718df66..3045b43 100644 --- a/scripts/human.py +++ b/scripts/human.py @@ -17,7 +17,6 @@ from scripts.utils import * D = bpy.data - # Load clothes list with open(r"mh_models\mh_mass_produce.json") as f: data = json.load(f) @@ -97,7 +96,8 @@ class Human: bpy.ops.object.mode_set(mode='POSE') ik_constraints = { 'upperarm': [(-45, 120), (-90, 90), (-80, 80)], - 'lowerarm': [(-30, 120), None, None] + 'lowerarm': [(-30, 120), None, None], + 'clavicle': [None, None, None] } for s in ('l', 'r'): lowerarm = self.model.pose.bones[f'lowerarm_{s}'] @@ -106,6 +106,12 @@ class Human: lowerarm.constraints['IK'].subtarget = f'hand_{s}_IK' lowerarm.constraints['IK'].chain_count = 2 + hand = self.model.pose.bones[f'hand_{s}'] + hand.constraints.new("COPY_ROTATION") + hand.constraints['Copy Rotation'].target = self.model + hand.constraints['Copy Rotation'].enabled = False + hand.constraints['Copy Rotation'].subtarget = f'hand_{s}_IK' + for name, consts in ik_constraints.items(): bone = self.model.pose.bones[f'{name}_{s}'] for axe, const in zip(('x', 'y', 'z'), consts): @@ -120,6 +126,9 @@ class Human: # self.model.pose.bones[f'hand_{s}_IK'].location = Vector((-15, 10, 0)) self.model.pose.bones[f'hand_{s}_IK'].location = Vector((0, 5, -10)) + for i in '123': + self.model.pose.bones[f'spine_0{i}'].lock_ik_z = True + bpy.ops.object.mode_set(mode='OBJECT') def init_textures(self): @@ -155,6 +164,15 @@ class Human: return self.model +def switch_constraints(model, enable=False): + for s in 'lr': + hand_ik = model.pose.bones[f'hand_{s}_IK'] + for constr in hand_ik.constraints: + constr.enabled = enable + model.pose.bones[f'hand_{s}'].constraints['Copy Rotation'].enabled = not enable + model.pose.bones[f'lowerarm_{s}'].constraints['IK'].chain_count = 2 + + def set_bounds(model, car=None): # set_floors(model, car) set_shrinkwraps(model, car) @@ -165,6 +183,7 @@ def set_floors(model, car=None): utils.select_only(model) bpy.ops.object.mode_set(mode='POSE') + planes = None if car is not None: planes = [ch for ch in car.children_recursive if ch.name[:5] == 'Plane'] @@ -188,8 +207,9 @@ def set_shrinkwraps(model, car=None): utils.select_only(model) bpy.ops.object.mode_set(mode='POSE') + out_objects = None if car is not None: - out_objects = [ch for ch in car.children_recursive if ch.name[:4] == 'OUT_'] + out_objects = [ch for ch in car.children_recursive if (ch.name[:4] == 'OUT_' or ch.name[:3] == 'IN_')] for s in 'lr': bone = model.pose.bones[f'hand_{s}_IK'] @@ -199,7 +219,7 @@ def set_shrinkwraps(model, car=None): for obj in out_objects: constr = bone.constraints.new('SHRINKWRAP') constr.target = obj - constr.wrap_mode = 'OUTSIDE' + constr.wrap_mode = 'OUTSIDE' if obj.name[:4] == 'OUT_' else 'INSIDE' if 'Back' in obj.name: constr.distance = model.pose.bones['lowerarm_l'].length * model.scale.x / obj.scale.y * 1 if 'Side' in obj.name: diff --git a/scripts/random_pose.py b/scripts/random_pose.py index 227f336..d4a7847 100644 --- a/scripts/random_pose.py +++ b/scripts/random_pose.py @@ -88,17 +88,25 @@ def reset_subject(subject): subject.rotation_euler = r([65, 0, 0]) -def hand_pose(pose): - for s in ['l', 'r']: +def hand_pose(pose, side, grasp=None): + if grasp is None: hand_ratio = random.uniform(0.1, 0.8) - for finger in ['thumb', 'index', 'middle', 'ring', 'pinky']: - angles = r([40, 40, 40]) if finger == 'thumb' else r([70, 90, 40]) - solo_ratio = random.uniform(0.7, 1) * hand_ratio - for i in range(3): - pose.bones[f'{finger}_{i + 1:02}_{s}'].rotation_euler.x = angles[i] * solo_ratio + elif grasp is True: + hand_ratio = random.uniform(0.5, 0.8) + elif grasp is False: + hand_ratio = random.uniform(0.05, 0.15) + + print(grasp) + print(hand_ratio) + for finger in ['thumb', 'index', 'middle', 'ring', 'pinky']: + angles = r([40, 40, 40]) if finger == 'thumb' else r([70, 90, 40]) + solo_ratio = random.uniform(0.7, 1) * hand_ratio + for i in range(3): + pose.bones[f'{finger}_{i + 1:02}_{side}'].rotation_euler.x = angles[i] * solo_ratio -def random_pose_ik(subject, auto_ik=False): + +def random_pose_ik(subject, auto_ik=False, targets=None): """ 1- reset and fix legs 2- randomize back @@ -106,6 +114,8 @@ def random_pose_ik(subject, auto_ik=False): 3- randomize neck (backward proportional to back bending) 4- move arms with IK :param subject: subject Object + :param auto_ik: use auto_ik option + :param targets: choose among fixed wrist targets :return: """ # 1 @@ -128,27 +138,30 @@ def random_pose_ik(subject, auto_ik=False): for bone, angles in base_rots.items(): pose.bones[bone].rotation_euler = angles - # 2 - pose.bones['spine_03'].rotation_euler = get_angles(bounds_vals['spine_03']) + if targets is None: + # 2 + pose.bones['spine_03'].rotation_euler = get_angles(bounds_vals['spine_03']) - # 2b Compensate for shoulder in seat by bending back to front - pose.bones['spine_01'].rotation_euler = r( - Vector((random.randint(0, 10) + max(d(abs(rota('spine_03').y)) - 20, 0) / 2, 0, - 0))) + rota('spine_01') + # 2b Compensate for shoulder in seat by bending back to front + pose.bones['spine_01'].rotation_euler = r( + Vector((random.randint(0, 10) + max(d(abs(rota('spine_03').y)) - 20, 0) / 2, 0, + 0))) + rota('spine_01') - # 3 - pose.bones['neck_01'].rotation_euler = ( - get_angles(bounds_vals['neck_01']) + - get_angles( - [[d((rota('spine_01').x + rota('spine_03').x) * -0.5), 0], None, None]) + - Vector((0, random.uniform(0, 0.5) * rota('spine_03').y, 0)) - ) + # 3 + pose.bones['neck_01'].rotation_euler = ( + get_angles(bounds_vals['neck_01']) + + get_angles( + [[d((rota('spine_01').x + rota('spine_03').x) * -0.5), 0], None, None]) + + Vector((0, random.uniform(0, 0.5) * rota('spine_03').y, 0)) + ) - hand_pose(pose) + else: + pose.bones['spine_03'].rotation_euler = get_angles([[5, 20], 15, None]) + pose.bones['neck_01'].rotation_euler = get_angles(bounds_vals['neck_01']) pose.use_auto_ik = auto_ik - targets = { + targets_test = { 'l': Vector((0.3, -0.1, 0.4)), 'r': Vector((-0.4, -0.1, 0.2)) } @@ -166,124 +179,11 @@ def random_pose_ik(subject, auto_ik=False): bone = pose_bone.bone select_only_bone(armature, bone) - target = Vector() - min_arm_factor = 0.2 - if False: - # target.x = random.random() * (0.4 if s == 'l' else -0.7) - target.x = random.uniform(max(-0.5, back_rota_fact * 0.4 + 0.4 if s == 'l' else 0), - min(0.4, back_rota_fact * 0.4 + 0 if s == 'l' else -0.4)) - target.z = random.uniform(0.05, 0.7) - if abs(target.x) < 0.2: - target.y = random.uniform(-0.04, 0.02) - else: - target.y = random.uniform(-0.24, 0.02) - # sloap - target.y -= random.random() * (target.z * 0.1) - elif False: - phi = random.uniform(r(-180 + 45), 0) - (r(45) if s == 'r' else 0) + rota('spine_03').y - costheta = random.uniform(-1, 1) - u = random.uniform(0.1, 1) - - theta = acos(costheta) - # u = arm_length * (u ** 1 / 3) - - target.x = u * sin(theta) * cos(phi) - target.y = u * sin(theta) * sin(phi) - target.z = u * cos(theta) - - shoulder_pose = get_head_pose(f'upperarm_{s}', subject) - target *= arm_length - target += shoulder_pose - - # target.x = max(min(target.x, 0.32), -0.55) - # target.y = max(min(target.y, 0.02), -0.04 if abs(target.x) < 0.2 else -0.24) - # target.z = max(min(target.z, 0.7), 0.15) - - bounded_width = Vector(((0.32 if s == 'l' else 0.55) + 0.35, - 2 * arm_length, - 0.7 - 0.15)) - target = ( - (target - shoulder_pose) * - (bounded_width / (2 * arm_length)) + - shoulder_pose + 0.5 * (bounded_width - Vector([2 * arm_length, ] * 3)) - ) - - # y done afterward because of the wheel - bounded_width_y = 0.02 - (- 0.04 if abs(target.x) < 0.2 else -0.24) - target.y = ( - (target.y - shoulder_pose.y) * - (bounded_width_y / (0.9 * arm_length)) + - shoulder_pose.y + 0.5 * (bounded_width_y - 0.9 * arm_length) - ) - elif False: - phi = random.uniform( - r(-180) + rota('spine_03').y if s == 'r' else r(-120) + rota('spine_03').y, - r(-60) + rota('spine_03').y if s == 'r' else r(-60) + rota('spine_03').y - ) - costheta = random.uniform(max(-0.8, 0.2 - cos(rota('spine_03').x + rota('spine_01').x)), - min(0.8, -0.2 + cos(rota('spine_03').x - rota('spine_01').x))) - - theta = acos(costheta) - bounds_u = [] + if targets is None: + target = Vector() shoulder_pose = get_head_pose(f'upperarm_{s}', subject) - if not sin(theta) * cos(phi): - min_u_x = 0 - max_u_x = arm_length - else: - bounds_u_x = ( - (-0.55 - shoulder_pose.x) / (sin(theta) * cos(phi)), - (0.32 - shoulder_pose.x) / (sin(theta) * cos(phi)) - ) - min_u_x = (-0.55 - shoulder_pose.x) / (sin(theta) * cos(phi)) - max_u_x = (0.32 - shoulder_pose.x) / (sin(theta) * cos(phi)) - bounds_u += [min_u_x, max_u_x] - - if not cos(theta): - min_u_z = 0 - max_u_z = arm_length - else: - bounds_u_z = ( - (0.15 - shoulder_pose.z) / cos(theta), - (0.7 - shoulder_pose.z) / cos(theta) - ) - min_u_z = (0.15 - shoulder_pose.z) / cos(theta) - max_u_z = (0.7 - shoulder_pose.z) / cos(theta) - - bounds_u += [min_u_z, max_u_z] - - max_u_xz = min([m for m in bounds_u + [arm_length] if m > min_arm_factor * arm_length]) - max_x = max_u_xz * sin(theta) * cos(phi) + shoulder_pose.x - - if not sin(theta) * sin(phi): - min_u_y = 0 - max_u_y = arm_length - else: - bounds_u_y = ( - ((-0.04 if abs(max_x) < 0.2 else -0.24) - shoulder_pose.y) / (sin(theta) * sin(phi)), - (0.02 - shoulder_pose.y) / (sin(theta) * sin(phi)) - ) - min_u_y = ((-0.04 if abs(max_x) < 0.2 else -0.24) - shoulder_pose.y) / (sin(theta) * sin(phi)) - max_u_y = (0.02 - shoulder_pose.y) / (sin(theta) * sin(phi)) - - bounds_u += [min_u_y, max_u_y] - - max_u = min([m for m in bounds_u + [arm_length] if m > min_arm_factor * arm_length]) - - # print(d(phi), d(theta)) - # print(min_u_x, min_u_y, min_u_z) - # print(max_u_x, max_u_y, max_u_z, max_u / arm_length) - - # u = random.uniform((min_arm_factor) ** 1/2, (max_u / arm_length) ** 1/2) ** 2 * arm_length - u = random.uniform(min_arm_factor * arm_length, max_u) - # print(u / arm_length) - - target.x = u * sin(theta) * cos(phi) - target.y = u * sin(theta) * sin(phi) - target.z = u * cos(theta) + min_arm_factor = 0.2 - target += shoulder_pose - - else: back_forward_angle = rota('spine_03').x + rota('spine_01').x - r(30) # 0 = straight phi = random.uniform( @@ -295,10 +195,7 @@ def random_pose_ik(subject, auto_ik=False): min(0.8, cos(theta_bound + back_forward_angle + rota('neck_01').x))) theta = acos(costheta) - bounds_u = [] - shoulder_pose = get_head_pose(f'upperarm_{s}', subject) - # u = random.uniform((min_arm_factor) ** 1/2, (max_u / arm_length) ** 1/2) ** 2 * arm_length min_arm_factor = 0.2 + max(sin(back_forward_angle), 0) # print(min_arm_factor, d(phi), d(theta)) u = random.uniform(min_arm_factor, 1) * arm_length @@ -309,14 +206,29 @@ def random_pose_ik(subject, auto_ik=False): target += shoulder_pose - temp_rota = rota(f'upperarm_{s}') + rota(f'lowerarm_{s}') - # target = targets[s] + hand_pose(pose, side=s) + + temp_rota = rota(f'upperarm_{s}') + rota(f'lowerarm_{s}') + # target = targets_test[s] + else: + target = random.choice(targets[s]) + pose.bones[f'hand_{s}_IK'].rotation_euler = Vector((0, 0, 0)) + pose.bones[f'hand_{s}_IK'].rotation_euler = ( + ((matrix_world @ pose.bones[f'hand_{s}_IK'].matrix).inverted() @ target.matrix_world).to_euler()) + print("_close" in target.name) + hand_pose(pose, side=s, grasp="_close" in target.name) + + target = target.location location = get_head_pose(bone.name, subject) # print(location) # print(target) bpy.ops.transform.translate(value=target - location) + if targets is not None: + for s in 'lr': + subject.pose.bones[f'lowerarm_{s}'].constraints['IK'].chain_count = 6 + bpy.ops.object.mode_set(mode='OBJECT') diff --git a/scripts/utils.py b/scripts/utils.py index 4320460..b905f50 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -43,7 +43,7 @@ def hide_object(obj, hide=True): hide_object(o, hide=hide) obj.hide_set(hide) obj.hide_select = hide - if 'Plane' in obj.name or 'OUT_' in obj.name: + if 'Plane' in obj.name or 'OUT_' in obj.name or 'IN_' in obj.name: obj.hide_render = True obj.hide_set(True) else: @@ -136,7 +136,7 @@ class Randomizer: def __call__(self, *args, **kwargs): pick_idx = random.randint(0, len(self) - 1) - return self.get(pick_idx) + return self.get(pick_idx, *args, **kwargs) def swap_object(self, obj=None): for o in self: -- GitLab