lds-processing/source2mets at main · lcnetdev/lds-processing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/usr/bin/env python

import yaml
import sys
import os

import glob

import datetime
from time import gmtime, strftime, sleep

from multiprocessing.dummy import Pool as ThreadPool
import itertools

from os import listdir
from os.path import isdir, isfile, join

from modules.config_parser import args

def Convert(config, j):
    config = config
    n = j["pos"]

    print("Processing job: " + str(n) + "; " + j["infile"] + "; " + j["outfile"])

    cmd = config["command"]
    cmd = cmd.replace('%INFILE%', j["infile"])
    cmd = cmd.replace('%OUTFILE%', j["outfile"])
    # print(cmd)
    os.system(cmd)

    # Usually move these, but they all end up with the same name day after day
    # so going to delete for now......
    os.unlink(j["infile"])

def dircontents(path, files):
    for f in listdir(path):
        fpath = join(path, f)
        if isfile(fpath) and fpath.endswith('.xml'):
            files.append(fpath)
        elif isdir(fpath):
            dircontents(fpath, files)
    return files

config = yaml.safe_load(open(args.config))
print()
print("Config:")
print(config)
print()

jobconfig = config["source2mets"]
print("Job config:")
print(jobconfig)
print()


files = []
files = dircontents(jobconfig["source_directory"], files)

#print(str(len(files)))
#files = files[:10]
#print(files)
#print(str(len(files)))
#print()
# sys.exit(0)

pos = 1
jobs = []
for f in files:

    infile = f
    outfile = jobconfig["target_directory"]
    j = {
        "pos": pos,
        "infile": infile,
        "outfile": outfile
    }
    jobs.append(j)
    pos += 1

if jobconfig["clean_target_directory"]:
    for f in glob.glob(jobconfig["target_directory"] + "*", recursive=False):
        if isfile(f):
            os.unlink(f)

print()
print("Number of threads: " + str(jobconfig["threads"]))
print("Total number of jobs: " + str(len(jobs)))
print()
st = datetime.datetime.now()
starttime = strftime("%Y-%m-%d %H:%M:%S", gmtime())

# make the Pool of workers
pool = ThreadPool(jobconfig["threads"])

# open the urls in their own threads
# and return the results
results = pool.starmap(Convert, zip(itertools.repeat(jobconfig), jobs))

# close the pool and wait for the work to finish
pool.close()
pool.join()

gt = gmtime()
endtime = strftime("%Y-%m-%d %H:%M:%S", gt)
et = datetime.datetime.now()
timedelta = et - st

print()
print()
print ("Task started at: " + starttime)
print ("Task ended at: " + endtime)
print ("Elapsed time: ", str(timedelta))
print()
print()