Crop point cloud around person

3dscan/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Astatin3
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

3dscan/main.py

@@ -0,0 +1,71 @@
+from threading import  Thread
+import cv2
+import numpy as np
+import ktb
+import open3d as o3d
+import math
+
+import skeleton
+
+# vis = o3d.visualization.Visualizer()
+# vis.create_window()
+
+# reconstruction = o3d.geometry.PointCloud()
+# reconstruction.points = o3d.utility.Vector3dVector(np.random.rand(2, 3))
+
+# vis.add_geometry(reconstruction)
+
+running = True
+
+def run_loop():
+    k = ktb.Kinect()
+
+    import socket
+    clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    clientsocket.connect(('localhost', 65000))
+    print("Connected!")
+
+    while running:
+        points, colors = k.get_ptcld(colorized=True, scale=10)
+        
+        mask = skeleton.calc_mask(k.get_frame(ktb.DEPTH), cv2.cvtColor(k.get_frame(ktb.COLOR), cv2.COLOR_BGR2RGB))
+        cv2.imshow('Person Mask', mask * 255)
+        cv2.waitKey(1)
+        # mask = mask.flatten().reshape((-1, 1))
+        mask = mask.flatten().astype(bool)
+        
+        points = points.reshape((-1, 3))
+        points = points[mask]
+        if points.shape[0] == 0:
+            continue
+        skip_count = math.ceil(points.shape[0]/2000)
+        points = points[0::skip_count]
+        points = np.trunc(points).astype(int)
+        
+        colors = colors.reshape((-1, 3))
+        colors = colors[mask]
+        colors = colors[0::skip_count]
+        colors *= 256
+        colors = np.trunc(colors).astype(int)
+        
+        # reconstruction.points = o3d.utility.Vector3dVector(points)
+        # reconstruction.colors = o3d.utility.Vector3dVector(colors)
+        # vis.update_geometry(reconstruction)
+        
+        for i in range(len(points)):
+            point = points[i]
+            color = colors[i]
+            clientsocket.send(f'{i},{point[0]},{point[1]},{point[2]},{color[0]},{color[1]},{color[2]}\n'.encode())
+ 
+            # print(f'{i},{(point[0])},{(point[1])},{(point[2])},{color[0]},{color[1]},{color[2]}')
+        print("Update!")
+        
+t = Thread(target=run_loop)
+t.start()
+
+# while running:
+#     running = vis.poll_events()
+#     vis.update_renderer()
+
+t.join()
+skeleton.pose.close()

3dscan/skeleton.py

@@ -0,0 +1,86 @@
+import cv2
+import numpy as np
+import ktb
+import mediapipe as mp
+
+from scipy.spatial import ConvexHull
+
+running = True
+
+# Initialize MediaPipe Pose
+mp_pose = mp.solutions.pose
+mp_drawing = mp.solutions.drawing_utils
+pose = mp_pose.Pose(static_image_mode=False, min_detection_confidence=0.5, min_tracking_confidence=0.5)
+
+
+def get_color():
+    return k.get_frame(ktb.COLOR)
+
+def get_depth_map():
+    return k.get_frame(ktb.DEPTH)
+
+def normalize_depth_map(depth_map):
+    min_val, max_val, _, _ = cv2.minMaxLoc(depth_map)
+    normalized = cv2.convertScaleAbs(depth_map, alpha=255.0/(max_val-min_val), beta=-min_val * 255.0/(max_val-min_val))
+    return normalized
+
+def calc_mask(depth_map, rgb_image):
+
+
+    # Process the image and get pose landmarks
+    results = pose.process(rgb_image)
+
+    if results.pose_landmarks:
+        # Create person mask
+        print("Found person!")
+        return create_person_mask(depth_map, results.pose_landmarks, rgb_image.shape)
+    else:
+        return np.zeros(rgb_image.shape[:2], dtype=np.uint8)
+
+
+def create_person_mask(depth_map, pose_landmarks, image_shape, distance_threshold=25):
+    # Create an empty mask
+    mask = np.zeros(image_shape[:2], dtype=np.uint8)
+    
+    # Draw pose landmarks and connections
+    for connection in mp_pose.POSE_CONNECTIONS:
+        start_point = pose_landmarks.landmark[connection[0]]
+        end_point = pose_landmarks.landmark[connection[1]]
+        
+        x1, y1 = int(start_point.x * image_shape[1]), int(start_point.y * image_shape[0])
+        x2, y2 = int(end_point.x * image_shape[1]), int(end_point.y * image_shape[0])
+        
+        cv2.line(mask, (x1, y1), (x2, y2), 1, thickness=distance_threshold*2)
+    
+    # Dilate the mask to include nearby pixels
+    kernel = np.ones((distance_threshold, distance_threshold), np.uint8)
+    mask = cv2.dilate(mask, kernel, iterations=1)
+    
+    # Get depth values for the pose landmarks
+    landmark_depths = []
+    for landmark in pose_landmarks.landmark:
+        x, y = int(landmark.x * image_shape[1]), int(landmark.y * image_shape[0])
+        if 0 <= x < image_shape[1] and 0 <= y < image_shape[0]:
+            landmark_depths.append(depth_map[y, x])
+    
+    # Calculate depth range
+    min_depth = np.percentile(landmark_depths, 0)  # 5th percentile to avoid outliers
+    max_depth = np.percentile(landmark_depths, 100)  # 95th percentile to avoid outliers
+    
+    # Refine the mask using depth information
+    depth_mask = (depth_map >= min_depth) & (depth_map <= max_depth)
+    
+    # Combine the initial mask with the depth mask
+    final_mask = mask & depth_mask
+    
+    return final_mask
+
+
+# while running:
+#     # vis.update_geometry(reconstruction)
+#     # print("E")
+#     if cv2.waitKey(1) & 0xFF == ord('q'):
+#         running = False
+#     running = vis.poll_events()
+#     vis.update_renderer()
+

run/logs/latest.log

@@ -1,70 +1,3 @@
-[17:38:34] [main/INFO] (FabricLoader/GameProvider) Loading Minecraft 1.21 with Fabric Loader 0.15.11
-[17:38:34] [main/INFO] (FabricLoader) Loading 42 mods:
-	- fabric-api 0.100.3+1.21
-	- fabric-api-base 0.4.42+6573ed8cd1
-	- fabric-api-lookup-api-v1 1.6.67+b5597344d1
-	- fabric-biome-api-v1 13.0.28+6fc22b99d1
-	- fabric-block-api-v1 1.0.22+0af3f5a7d1
-	- fabric-block-view-api-v2 1.0.10+6573ed8cd1
-	- fabric-command-api-v1 1.2.48+f71b366fd1
-	- fabric-command-api-v2 2.2.27+6a6dfa19d1
-	- fabric-commands-v0 0.2.65+df3654b3d1
-	- fabric-content-registries-v0 8.0.13+b5597344d1
-	- fabric-convention-tags-v1 2.0.14+7f945d5bd1
-	- fabric-convention-tags-v2 2.3.1+8a3b5617d1
-	- fabric-crash-report-info-v1 0.2.29+0af3f5a7d1
-	- fabric-data-attachment-api-v1 1.1.23+6a6dfa19d1
-	- fabric-data-generation-api-v1 20.2.8+16c4ae25d1
-	- fabric-dimensions-v1 4.0.0+6fc22b99d1
-	- fabric-entity-events-v1 1.6.12+6fc22b99d1
-	- fabric-events-interaction-v0 0.7.10+e633f883d1
-	- fabric-game-rule-api-v1 1.0.52+6573ed8cd1
-	- fabric-gametest-api-v1 2.0.1+6fc22b99d1
-	- fabric-item-api-v1 11.0.0+afdfc921d1
-	- fabric-item-group-api-v1 4.1.1+cb5ced13d1
-	- fabric-lifecycle-events-v1 2.3.11+8f3583aed1
-	- fabric-loot-api-v2 3.0.10+6573ed8cd1
-	- fabric-message-api-v1 6.0.13+6573ed8cd1
-	- fabric-networking-api-v1 4.2.0+ab7edbacd1
-	- fabric-object-builder-api-v1 15.1.11+d1321076d1
-	- fabric-particles-v1 4.0.2+6573ed8cd1
-	- fabric-recipe-api-v1 5.0.9+6573ed8cd1
-	- fabric-registry-sync-v0 5.0.22+ab7edbacd1
-	- fabric-rendering-data-attachment-v1 0.3.48+73761d2ed1
-	- fabric-rendering-fluids-v1 3.1.6+b5597344d1
-	- fabric-resource-conditions-api-v1 4.2.1+d153f344d1
-	- fabric-resource-loader-v0 1.1.4+cb5ced13d1
-	- fabric-screen-handler-api-v1 1.3.79+b5597344d1
-	- fabric-transfer-api-v1 5.1.14+b5597344d1
-	- fabric-transitive-access-wideners-v1 6.0.12+6573ed8cd1
-	- fabricloader 0.15.11
-	- java 22
-	- minecraft 1.21
-	- mixinextras 0.3.5
-	- modid 1.0.0
-[17:38:34] [main/INFO] (FabricLoader/Mixin) SpongePowered MIXIN Subsystem Version=0.8.5 Source=file:/home/astatin3/.gradle/caches/modules-2/files-2.1/net.fabricmc/sponge-mixin/0.13.3+mixin.0.8.5/9527e6b0d2449408958fd1302594dc65ec5ade9c/sponge-mixin-0.13.3+mixin.0.8.5.jar Service=Knot/Fabric Env=SERVER
-[17:38:35] [main/INFO] (FabricLoader/Mixin) Loaded Fabric development mappings for mixin remapper!
-[17:38:35] [main/INFO] (FabricLoader/Mixin) Compatibility level set to JAVA_17
-[17:38:35] [main/INFO] (FabricLoader/Mixin) Compatibility level set to JAVA_21
-[17:38:35] [main/INFO] (FabricLoader/MixinExtras|Service) Initializing MixinExtras via com.llamalad7.mixinextras.service.MixinExtrasServiceImpl(version=0.3.5).
-[17:38:39] [main/INFO] (Minecraft) Environment: Environment[sessionHost=https://sessionserver.mojang.com, servicesHost=https://api.minecraftservices.com, name=PROD]
-[17:38:40] [main/INFO] (Minecraft) Loaded 1290 recipes
-[17:38:40] [main/INFO] (Minecraft) Loaded 1399 advancements
-[17:38:40] [main/INFO] (BiomeModificationImpl) Applied 0 biome modifications to 0 of 64 new biomes in 1.180 ms
-[17:38:40] [Server thread/INFO] (Minecraft) Starting minecraft server version 1.21
-[17:38:40] [Server thread/INFO] (Minecraft) Loading properties
-[17:38:40] [Server thread/INFO] (Minecraft) Default game type: SURVIVAL
-[17:38:40] [Server thread/INFO] (Minecraft) Generating keypair
-[17:38:40] [Server thread/INFO] (Minecraft) Starting Minecraft server on *:25565
-[17:38:40] [Server thread/INFO] (Minecraft) Using epoll channel type
-[17:38:40] [Server thread/INFO] (Minecraft) Preparing level "world"
-[17:38:40] [Point Cloud Thread/INFO] (Minecraft) [STDOUT]: Server is listening on port 65000
-[17:38:41] [Server thread/INFO] (Minecraft) Preparing start region for dimension minecraft:overworld
-[17:38:41] [Worker-Main-8/INFO] (Minecraft) Preparing spawn area: 0%
-[17:38:41] [Server thread/INFO] (Minecraft) Time elapsed: 252 ms
-[17:38:41] [Server thread/INFO] (Minecraft) Done (1.079s)! For help, type "help"
-[17:38:42] [User Authenticator #1/INFO] (Minecraft) UUID of player ASTATIN3 is 1451d26e-5141-4801-8e94-c35164f87be9
-[17:38:43] [Server thread/INFO] (Minecraft) ASTATIN3[/127.0.0.1:42146] logged in with entity id 20 at (0.13286859306182097, 126.31511469407461, -1.0259515056122512)
-[17:38:43] [Server thread/INFO] (Minecraft) ASTATIN3 joined the game
-[17:38:46] [Point Cloud Thread/INFO] (Minecraft) [STDOUT]: New client connected
-[17:39:14] [Server thread/INFO] (Minecraft) [ASTATIN3: Set the time to 1000]
+[11:01:29] [Server thread/WARN] (Minecraft) Can't keep up! Is the server overloaded? Running 2317ms or 46 ticks behind
+[11:02:15] [Server thread/INFO] (Minecraft) ASTATIN3 lost connection: Disconnected
+[11:02:15] [Server thread/INFO] (Minecraft) ASTATIN3 left the game

run/server.properties

@@ -1,5 +1,5 @@
 #Minecraft server properties
-#Sat Aug 17 17:38:39 MDT 2024
+#Sat Aug 17 22:12:35 MDT 2024
 accepts-transfers=false
 allow-flight=false
 allow-nether=true

run/usercache.json

@@ -1 +1 @@
-[{"name":"ASTATIN3","uuid":"1451d26e-5141-4801-8e94-c35164f87be9","expiresOn":"2024-09-17 17:38:43 -0600"}]
+[{"name":"ASTATIN3","uuid":"1451d26e-5141-4801-8e94-c35164f87be9","expiresOn":"2024-09-17 22:12:43 -0600"}]

run/world/stats/1451d26e-5141-4801-8e94-c35164f87be9.json

@@ -1 +1 @@
-{"stats":{"minecraft:custom":{"minecraft:time_since_rest":129902,"minecraft:leave_game":14,"minecraft:play_time":129902,"minecraft:time_since_death":129902,"minecraft:sneak_time":870,"minecraft:total_world_time":129902,"minecraft:fly_one_cm":258353},"minecraft:used":{"minecraft:grass_block":4}},"DataVersion":3953}
+{"stats":{"minecraft:dropped":{"minecraft:grass_block":1},"minecraft:custom":{"minecraft:time_since_rest":171824,"minecraft:leave_game":15,"minecraft:play_time":171824,"minecraft:time_since_death":171824,"minecraft:sneak_time":994,"minecraft:total_world_time":171824,"minecraft:drop":1,"minecraft:fly_one_cm":263255},"minecraft:used":{"minecraft:grass_block":4}},"DataVersion":3953}

src/main/java/com/example/testThread.java

@@ -1,34 +1,25 @@
 package com.example;
 
-import com.mojang.serialization.*;
-import it.unimi.dsi.fastutil.objects.Reference2ObjectArrayMap;
-import net.minecraft.block.BlockState;
-import net.minecraft.block.Blocks;
 import net.minecraft.particle.DustParticleEffect;
-import net.minecraft.particle.ParticleEffect;
-import net.minecraft.particle.ParticleType;
-import net.minecraft.particle.ParticleTypes;
 import net.minecraft.registry.RegistryKey;
 import net.minecraft.server.world.ServerWorld;
-import net.minecraft.util.math.BlockPos;
 import net.minecraft.util.math.Vec3d;
-import net.minecraft.util.math.Vec3i;
 import net.minecraft.world.World;
-import org.joml.Vector3f;
 
 import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.OutputStream;
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 
 public class testThread extends Thread {
     private Map<RegistryKey<World>, ServerWorld> worlds;
-    private short[][] points = new short[2300][6];
+    private short[][] points = new short[2000][6];
 
     BufferedReader reader;
 //    PrintWriter writer;
@@ -100,6 +91,11 @@ public class testThread extends Thread {
     }
 
     public void run() {
+        Integer[] array = new Integer[points.length];
+        Arrays.setAll(array, i -> i);
+        List<Integer> list = Arrays.asList(array);
+        Collections.shuffle(list);
+
         while(true){
             try {
                 Thread.sleep(100);
@@ -110,20 +106,21 @@ public class testThread extends Thread {
 
             ServerWorld world = worlds.get(World.OVERWORLD);
 
-            for(int i=0; i < points.length; i++) {
-                if(points[i] == null) {continue;}
-//                System.out.println(points[i][0]/100. + ", " + points[i][1]/100. + ", " + points[i][2]/100.);
+            for(int i = 0; i < points.length; i++) {
+                int index = list.get(i);
+                if(points[index] == null) {continue;}
+//                System.out.println(points[index][0]/100. + ", " + points[index][1]/100. + ", " + points[index][2]/100.);
 //
-                int rgb =          Math.clamp(points[i][5], 0, 255);
-                rgb = (rgb << 8) + Math.clamp(points[i][4], 0, 255);
-                rgb = (rgb << 8) + Math.clamp(points[i][3], 0, 255);
+                int rgb =          Math.clamp(points[index][5], 0, 255);
+                rgb = (rgb << 8) + Math.clamp(points[index][4], 0, 255);
+                rgb = (rgb << 8) + Math.clamp(points[index][3], 0, 255);
 
                 world.spawnParticles(
                         new DustParticleEffect(Vec3d.unpackRgb(rgb).toVector3f(), 0.2F),
 //                        new DustParticleEffect(DustParticleEffect.RED, 0.5F),
-                        points[i][0]*0.02,
-                        (points[i][2]*0.02) + 128,
-                        points[i][1]*0.02,
+                        points[index][0]*0.02,
+                        (points[index][2]*0.02) + 128,
+                        points[index][1]*0.02,
                         10,
                         0,
                         0,