MihaiPopa-1/minecraft-skins-1.1m-deduped-64x64

--- task_categories: - text-to-image - image-classification - unconditional-image-generation tags: - minecraft - minecraft-skins - de-duped - deduped - zip-dataset - zip - zip-archive size_categories: - 1M<n<10M license: apache-2.0 pretty_name: Minecraft Skins 1.1M Deduped (64x64 Edition) --- # Minecraft Skins 1.1M Deduped (64x64 Edition)! [Nyuuzyou's Minecraft-Skins-20M](https://huggingface.co/datasets/nyuuzyou/Minecraft-Skins-20M) but deduped using BLAKE3 hashes (only catches if pixel values are exactly the same), then filtered to only valid 64x64 skins. Format is a 2.6 GB ZIP archive containing 64x64 PNG skin files. # Tools used PIL Image (Python), Google Colab (free CPU tier) and BLAKE3 (Python) # How it was made 1. Loaded [Nyuuzyou's Minecraft-Skins-20M](https://huggingface.co/datasets/nyuuzyou/Minecraft-Skins-20M), 2. Deduped using BLAKE3, only catching if pixel values are exactly the same. Tiny differences are kept, 3. Result is 1296966 unique skins. 4. Invalid, 64x32 skins or other skin sizes are removed, 5. Result is 1107411 64x64 skins. 6. Output is given in a 2.6 GB ZIP archive. This can be used to make your own skin generation model (but I'm going with VQ-VAE anyway!) # Future improvements for version 2 1. Captioning (with Florence 2 Base) 2. Filtering troll skins (skins that are formed of just a single color) # Code Code to reproduce it (all by Claude 4.6 Sonnet): ```python # ============================================================ # Minecraft Skin Deduplicator # Downloads nyuuzyou/Minecraft-Skins-20M, hashes pixel data # with BLAKE3, deduplicates via SQLite, saves unique skins. # ============================================================ # --- 1. Install dependencies --- !pip install blake3 datasets Pillow numpy -q import blake3 import numpy as np import sqlite3 import os from PIL import Image from datasets import load_dataset import base64 from io import BytesIO # --- 2. Config --- OUTPUT_DIR = "/content/unique_skins" # where to save unique skins DB_PATH = "/content/hashes.db" # SQLite dedup DB (move to Drive to persist!) MOJANG_DEFAULT_PATHS = [] # optional: put paths to Steve/Alex/etc. here os.makedirs(OUTPUT_DIR, exist_ok=True) # --- 3. Load known Mojang default hashes (optional but recommended) --- def hash_pixels(img: Image.Image) -> bytes: """Hash the raw pixel data of a PIL image. Returns 32-byte BLAKE3 digest.""" arr = np.array(img.convert("RGBA")) return blake3.blake3(arr.tobytes()).digest() blacklist = set() for path in MOJANG_DEFAULT_PATHS: img = Image.open(path) blacklist.add(hash_pixels(img)) print(f"Blacklisted {len(blacklist)} default Mojang skins.") # --- 4. Set up SQLite --- conn = sqlite3.connect(DB_PATH) conn.execute("CREATE TABLE IF NOT EXISTS seen (h BLOB PRIMARY KEY)") conn.execute("PRAGMA journal_mode=WAL") # faster concurrent writes conn.execute("PRAGMA synchronous=NORMAL") # safe but faster than FULL conn.commit() def already_seen(digest: bytes) -> bool: return conn.execute("SELECT 1 FROM seen WHERE h=?", (digest,)).fetchone() is not None def mark_seen(digest: bytes): conn.execute("INSERT OR IGNORE INTO seen VALUES (?)", (digest,)) # --- 5. Stream & deduplicate --- print("Loading dataset (streaming)...") dataset = load_dataset( "nyuuzyou/Minecraft-Skins-20M", split="train", streaming=True, # <-- key: don't download everything at once ) total = 0 duplicates = 0 blacklisted = 0 saved = 0 BATCH_SIZE = 500 # commit to SQLite every N rows for i, row in enumerate(dataset): total += 1 # The image field — adjust key if the dataset uses a different column name raw = row.get("image") or row.get("skin") or row.get("img") if raw is None: continue # Handle both PIL images and raw bytes try: if isinstance(raw, str): img = Image.open(BytesIO(base64.b64decode(raw))) elif isinstance(raw, bytes): img = Image.open(BytesIO(raw)) else: img = raw # already a PIL Image except Exception: continue # skip corrupt/malformed entries digest = hash_pixels(img) if digest in blacklist: blacklisted += 1 continue if already_seen(digest): duplicates += 1 continue # Save unique skin out_path = os.path.join(OUTPUT_DIR, f"skin_{saved:08d}.png") img.save(out_path, format="PNG") mark_seen(digest) saved += 1 # Batch commit if total % BATCH_SIZE == 0: conn.commit() # Progress if total % 10_000 == 0: print(f" Processed: {total:,} | Saved: {saved:,} | Dupes: {duplicates:,} | Blacklisted: {blacklisted:,}") conn.commit() conn.close() print("\n=== Done! ===") print(f"Total processed : {total:,}") print(f"Unique skins : {saved:,}") print(f"Duplicates : {duplicates:,}") print(f"Blacklisted : {blacklisted:,}") print(f"Reduction : {100 * (1 - saved/max(total,1)):.1f}%") ``` then: ```python import os import zipfile from PIL import Image from tqdm import tqdm INPUT_DIR = "/content/unique_skins" OUTPUT_DIR = "/content/filtered_skins" ZIP_PATH = "/content/minecraft_skins_64x64.zip" os.makedirs(OUTPUT_DIR, exist_ok=True) # Step 1: Filter to 64x64 only print("Filtering to 64x64...") all_skins = os.listdir(INPUT_DIR) filtered = 0 skipped = 0 for filename in tqdm(all_skins): if not filename.endswith(".png"): continue path = os.path.join(INPUT_DIR, filename) try: img = Image.open(path) if img.size == (64, 64): img.close() filtered += 1 else: os.remove(path) # delete non-64x64 skipped += 1 except Exception: os.remove(path) # delete corrupt files skipped += 1 print(f"Kept: {filtered:,} | Skipped/removed: {skipped:,}") # Step 2: Pack into ZIP print("\nPacking into ZIP...") skin_files = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")] with zipfile.ZipFile(ZIP_PATH, "w", zipfile.ZIP_DEFLATED, compresslevel=1) as zf: for filename in tqdm(skin_files): zf.write(os.path.join(INPUT_DIR, filename), arcname=filename) print(f"\nDone! ZIP saved to {ZIP_PATH}") print(f"ZIP size: {os.path.getsize(ZIP_PATH) / 1024 / 1024:.1f} MB") ```### 任务类别 - 文本到图像（text-to-image） - 图像分类（image-classification） - 无条件图像生成（unconditional-image-generation） ### 标签 - 我的世界（Minecraft） - 我的世界皮肤（minecraft-skins） - 去重（de-duped） - 去重（deduped） - ZIP数据集（zip-dataset） - ZIP（zip） - ZIP归档（zip-archive） ### 数据量类别 - 100万 < 样本数 < 1000万 ### 许可证 Apache-2.0 ### 美观名称 110万去重版我的世界皮肤（64×64分辨率版） # 110万去重版我的世界皮肤（64×64分辨率版）！ 本数据集源自[Nyuuzyou发布的《我的世界2000万皮肤数据集》](https://huggingface.co/datasets/nyuuzyou/Minecraft-Skins-20M)，通过BLAKE3哈希算法进行去重（仅保留像素值完全一致的样本），并筛选出仅符合64×64分辨率的有效皮肤文件。 数据集格式为2.6GB的ZIP归档文件，内含64×64分辨率的PNG格式皮肤文件。 ## 所用工具 Python图像库（PIL Image）、谷歌Colab（Google Colab，免费CPU算力版本）以及BLAKE3（Python库） ## 数据集制作流程 1. 加载[Nyuuzyou发布的《我的世界2000万皮肤数据集》](https://huggingface.co/datasets/nyuuzyou/Minecraft-Skins-20M)； 2. 使用BLAKE3哈希算法对皮肤进行去重，仅过滤像素值完全一致的重复样本，细微差异的样本将被保留； 3. 去重后得到1296966个唯一皮肤样本； 4. 移除不符合要求的皮肤（如64×32分辨率或其他非标准尺寸的皮肤）； 5. 最终得到1107411个64×64分辨率的有效皮肤样本； 6. 将最终数据集打包为2.6GB的ZIP归档文件。 本数据集可用于构建自定义皮肤生成模型（本项目后续将采用VQ-VAE架构进行开发）。 ## 版本2的未来改进方向 1. 为皮肤添加文本标注（使用Florence 2 Base模型）； 2. 过滤恶意/无效皮肤（如仅由单一纯色构成的皮肤）。 ## 复现代码 以下为数据集复现代码（全部由Claude 4.6 Sonnet生成）： python # ============================================================ # 我的世界皮肤去重工具 # 下载nyuuzyou/Minecraft-Skins-20M数据集，使用BLAKE3对像素数据进行哈希，通过SQLite完成去重，并保存唯一皮肤样本。 # ============================================================ # --- 1. 安装依赖项 --- !pip install blake3 datasets Pillow numpy -q import blake3 import numpy as np import sqlite3 import os from PIL import Image from datasets import load_dataset import base64 from io import BytesIO # --- 2. 配置参数 --- OUTPUT_DIR = "/content/unique_skins" # 唯一皮肤的保存路径 DB_PATH = "/content/hashes.db" # SQLite去重数据库（可迁移至云端硬盘以持久化存储） MOJANG_DEFAULT_PATHS = [] # 可选：在此处添加Steve/Alex等官方默认皮肤的路径 os.makedirs(OUTPUT_DIR, exist_ok=True) # --- 3. 加载Mojang官方默认皮肤哈希值（可选但推荐） --- def hash_pixels(img: Image.Image) -> bytes: """对PIL图像的原始像素数据进行哈希，返回32字节的BLAKE3摘要。""" arr = np.array(img.convert("RGBA")) return blake3.blake3(arr.tobytes()).digest() blacklist = set() for path in MOJANG_DEFAULT_PATHS: img = Image.open(path) blacklist.add(hash_pixels(img)) print(f"已加入{len(blacklist)}个官方默认皮肤的黑名单。") # --- 4. 初始化SQLite数据库 --- conn = sqlite3.connect(DB_PATH) conn.execute("CREATE TABLE IF NOT EXISTS seen (h BLOB PRIMARY KEY)") conn.execute("PRAGMA journal_mode=WAL") # 启用预写日志以提升并发写入性能 conn.execute("PRAGMA synchronous=NORMAL") # 采用平衡的安全与性能模式 conn.commit() def already_seen(digest: bytes) -> bool: return conn.execute("SELECT 1 FROM seen WHERE h=?", (digest,)).fetchone() is not None def mark_seen(digest: bytes): conn.execute("INSERT OR IGNORE INTO seen VALUES (?)", (digest,)) # --- 5. 流式加载并去重数据集 --- print("正在流式加载数据集...") dataset = load_dataset( "nyuuzyou/Minecraft-Skins-20M", split="train", streaming=True, # 关键配置：无需一次性下载全部数据集 ) total = 0 duplicates = 0 blacklisted = 0 saved = 0 BATCH_SIZE = 500 # 每处理500条数据后提交一次SQLite事务 for i, row in enumerate(dataset): total += 1 # 读取图像字段：根据数据集列名调整字段键名 raw = row.get("image") or row.get("skin") or row.get("img") if raw is None: continue # 处理PIL图像与原始字节两种数据格式 try: if isinstance(raw, str): img = Image.open(BytesIO(base64.b64decode(raw))) elif isinstance(raw, bytes): img = Image.open(BytesIO(raw)) else: img = raw # 已为PIL图像格式 except Exception: continue # 跳过损坏或格式错误的样本 digest = hash_pixels(img) if digest in blacklist: blacklisted += 1 continue if already_seen(digest): duplicates += 1 continue # 保存唯一皮肤样本 out_path = os.path.join(OUTPUT_DIR, f"skin_{saved:08d}.png") img.save(out_path, format="PNG") mark_seen(digest) saved += 1 # 批量提交事务 if total % BATCH_SIZE == 0: conn.commit() # 进度打印 if total % 10_000 == 0: print(f" 已处理：{total:,} | 已保存：{saved:,} | 重复样本：{duplicates:,} | 已过滤黑名单样本：{blacklisted:,}") conn.commit() conn.close() print(" === 处理完成！ ===") print(f"总处理样本数：{total:,}") print(f"唯一皮肤样本数：{saved:,}") print(f"重复样本数：{duplicates:,}") print(f"已过滤黑名单样本数：{blacklisted:,}") print(f"数据压缩率：{100 * (1 - saved/max(total,1)):.1f}%") python import os import zipfile from PIL import Image from tqdm import tqdm INPUT_DIR = "/content/unique_skins" OUTPUT_DIR = "/content/filtered_skins" ZIP_PATH = "/content/minecraft_skins_64x64.zip" os.makedirs(OUTPUT_DIR, exist_ok=True) # 步骤1：筛选64×64分辨率的皮肤样本 print("正在筛选64×64分辨率皮肤...") all_skins = os.listdir(INPUT_DIR) filtered = 0 skipped = 0 for filename in tqdm(all_skins): if not filename.endswith(".png"): continue path = os.path.join(INPUT_DIR, filename) try: img = Image.open(path) if img.size == (64, 64): img.close() filtered += 1 else: os.remove(path) # 删除非64×64分辨率的样本 skipped += 1 except Exception: os.remove(path) # 删除损坏的文件 skipped += 1 print(f"保留样本数：{filtered:,} | 已移除/跳过样本数：{skipped:,}") # 步骤2：打包为ZIP归档文件 print(" 正在打包为ZIP归档...") skin_files = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")] with zipfile.ZipFile(ZIP_PATH, "w", zipfile.ZIP_DEFLATED, compresslevel=1) as zf: for filename in tqdm(skin_files): zf.write(os.path.join(INPUT_DIR, filename), arcname=filename) print(f" 打包完成！ZIP归档保存至：{ZIP_PATH}") print(f"ZIP归档大小：{os.path.getsize(ZIP_PATH) / 1024 / 1024:.1f} MB")