CurseMetaDB/CleanInjest.py at master · OpenMineMods/CurseMetaDB · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# CleanInject.py data/addons/ output.json

from sys import argv
from os import listdir, path
from json import loads, dumps
from MessCleaner import clean_category, clean_file, clean_project


def ls(folder: str):
    return [path.join(folder, i) for i in listdir(folder)]


folder = argv[1]
projects = dict()
categories = dict()
files = dict()

for project_folder in ls(folder):
    manifest = loads(open(path.join(project_folder, "index.json"), encoding='utf-8').read())
    project = clean_project(manifest)
    project["minecraft"] = list()
    project["categories"] = list()
    project["authors"] = list()

    pid = project["id"]

    cleaned_files = list()
    raw_files = ls(path.join(project_folder, "files"))

    for file in raw_files:
        clean = clean_file(loads(open(file, encoding='utf-8').read()))
        if clean is not None:
            cleaned_files.append(clean)

    for file in cleaned_files:
        fid = file["id"]
        file["project"] = pid
        project["minecraft"] += file["minecraft"]
        project["files"].append(fid)
        files[fid] = file

    oldest_date = min(file["date"] for file in cleaned_files)
    newest_date = max(file["date"] for file in cleaned_files)

    project["stats"] = {
        "downloads": int(manifest["DownloadCount"]),
        "popularity": float(manifest["PopularityScore"]),
        "created": int(oldest_date),
        "latest": int(newest_date)
    }

    for category in manifest["Categories"]:
        category = clean_category(category)
        cid = category["id"]
        categories[cid] = category
        project["categories"].append(cid)

    for author in manifest["Authors"]:
        project["authors"].append({"username": author["Name"]})

    if "Attachments" in manifest:
        for attachment in manifest["Attachments"]:
            if "IsDefault" in attachment and attachment["IsDefault"]:
                project["icon"] = {"url": attachment["Url"], "hash": path.splitext(attachment["Title"])[0]}
        if "icon" not in project:
            attachment = manifest["Attachments"][0]
            project["icon"] = {"url": attachment["Url"], "hash": path.splitext(attachment["Title"])[0]}

    project["minecraft"] = list(set(project["minecraft"]))
    projects[pid] = project

out = dumps({
    "projects": projects,
    "categories": categories,
    "files": files
}, separators=(",", ":"), indent=4, sort_keys=True)

f_len = open(argv[2], 'w', encoding='utf-8').write(out)
print("Wrote {} bytes to {}".format(f_len, argv[2]))