-
Notifications
You must be signed in to change notification settings - Fork 10
/
supertransfer2.sh
201 lines (180 loc) · 8.38 KB
/
supertransfer2.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
############################################################################
# INIT
############################################################################
echo -e " [INFO] Initializing Supertransfer2 Load Balanced Multi-SA Uploader..."
source /opt/plexguide/scripts/supertransfer/rcloneupload.sh
source /opt/plexguide/scripts/supertransfer/init.sh
source /opt/plexguide/scripts/supertransfer/settings.conf
source ${userSettings}
#dbug=on
# check to make sure filepaths are there
touch /tmp/superTransferUploadSuccess &>/dev/null
touch /tmp/superTransferUploadFail &>/dev/null
[[ -e $gdsaDB ]] || touch $gdsaDB &>/dev/null
[[ -e $uploadHistory ]] || touch $uploadHistory &>/dev/null
[[ -d $jsonPath ]] || mkdir $jsonPath &>/dev/null
[[ -d $logDir ]] || mkdir $logDir &>/dev/null
[[ ! -e $userSettings ]] && echo -e " [FAIL] No User settings found in $userSettings. Exiting." && exit 1
clean_up(){
echo -e " [INFO] SIGINT: Clearing filelocks and logs. Exiting."
numSuccess=$(cat /tmp/superTransferUploadSuccess | wc -l)
numFail=$(cat /tmp/superTransferUploadFail | wc -l)
totalUploaded=$(awk -F'=' '{ sum += $2 } END { print sum / 1000000 }' $gdsaDB)
sizeLeft=$(du -hc ${localDir} | tail -1 | awk '{print $1}')
echo -e " [STAT]\t$numSuccess Successes, $numFail Failures, $sizeLeft left in $localDir, ${totalUploaded}GB total uploaded"
rm ${logDir}/* &>/dev/null
echo -n '' > ${fileLock}
# if user added or removed GDSA remotes, reset the usage database in order to regenerate it
numGdsaDB=$(cat ${gdsaDB} | wc -l)
numGdsa=$(rclone listremotes --config=/root/.config/rclone/rclone.conf | wc -l)
[[ $numGdsaDB == $numGdsa ]] || echo -n '' > ${gdsaDB}
rm /tmp/superTransferUploadFail &>/dev/null
rm /tmp/superTransferUploadSuccess &>/dev/null
rm /tmp/.SA_error.log.tmp &>/dev/null
rm /tmp/SA_error.log &>/dev/null
exit 0
}
trap "clean_up" SIGINT
trap "clean_up" SIGTERM
############################################################################
# Initalize gdsaDB (can be skipped with --skip)
############################################################################
init_DB(){
# get list of avail gdsa accounts
gdsaList=$(rclone listremotes --config /root/.config/rclone/rclone.conf | sed 's/://' | egrep '^GDSA[0-9]+$')
if [[ -n $gdsaList ]]; then
numGdsa=$(echo $gdsaList | wc -w)
echo -e " [INFO] Initializing $numGdsa Service Accounts."
else
echo "[INFO] No GDSA's found in /root/.config/rclone/rclone.conf, checking ~/.config/rclone/rclone.conf for GDSA's"
# backup root's rclone conf
[[ -e /root/.config/rclone/rclone.conf ]] && cp /root/.config/rclone/rclone.conf /root/.config/rclone/rclone.conf.back
# cp home's rclone conf to root
[[ -e ~/.config/rclone/rclone.conf ]] && cp ~/.config/rclone/rclone.conf /root/.config/rclone/rclone.conf
gdsaList=$(rclone listremotes --config /root/.config/rclone/rclone.conf | sed 's/://' | egrep '^GDSA[0-9]+$')
[[ -z $gdsaList ]] && echo -e " [FAIL] No Valid SA accounts found! Is Rclone Configured With GDSA## remotes?" && exit 1
echo "[INFO] Found GDSA's in ~/ and copying to /root"
numGdsa=$(echo $gdsaList | wc -w)
echo -e " [INFO] Initializing $numGdsa Service Accounts."
fi
# reset existing logs & db
echo -n '' > /tmp/SA_error.log
validate(){
local s=0
rclone lsd --config /root/.config/rclone/rclone.conf ${1}:/ &>/tmp/.SA_error.log.tmp && s=1
if [[ $s == 1 ]]; then
echo -e " [ OK ] ${1}\t Validation Successful!"
egrep -q ^${1}=. $gdsaDB || echo "${1}=0" >> $gdsaDB
else
echo -e " [WARN] ${1}\t Validation FAILURE!"
cat /tmp/.SA_error.log.tmp >> /tmp/SA_error.log
((gdsaFail++))
fi
}
i=0
numProcs=10
# parallelize validator for speeeeeed
for gdsa in $gdsaList; do
if (( i++ >= numProcs )); then
wait -n
fi
validate $gdsa &
sleep 0.1
done
wait
gdsaLeast=$(sort -gr -k2 -t'=' ${gdsaDB} | egrep ^GDSA[0-9]+=. | tail -1 | cut -f1 -d'=')
[[ -n $gdsaFail ]] && echo -e " [WARN] $gdsaFail Failure(s). See /tmp/SA_error.log"
}
init_DB_encrypt(){
# get list of avail gdsa accounts
gdsaList=$(rclone listremotes --config /root/.config/rclone/rclone.conf | sed 's/://' | egrep '^GDSAC[0-9]+$')
if [[ -n $gdsaList ]]; then
numGdsa=$(echo $gdsaList | wc -w)
echo -e " [INFO] Initializing $numGdsa Service Accounts."
else
echo "[INFO] No GDSA's found in /root/.config/rclone/rclone.conf, checking ~/.config/rclone/rclone.conf for GDSA's"
# backup root's rclone conf
[[ -e /root/.config/rclone/rclone.conf ]] && cp /root/.config/rclone/rclone.conf /root/.config/rclone/rclone.conf.back
# cp home's rclone conf to root
[[ -e ~/.config/rclone/rclone.conf ]] && cp ~/.config/rclone/rclone.conf /root/.config/rclone/rclone.conf
gdsaList=$(rclone listremotes --config /root/.config/rclone/rclone.conf | sed 's/://' | egrep '^GDSAC[0-9]+$')
[[ -z $gdsaList ]] && echo -e " [FAIL] No Valid SA accounts found! Is Rclone Configured With GDSA## remotes?" && exit 1
echo "[INFO] Found GDSA's in ~/ and copying to /root"
numGdsa=$(echo $gdsaList | wc -w)
echo -e " [INFO] Initializing $numGdsa Service Accounts."
fi
# reset existing logs & db
echo -n '' > /tmp/SA_error.log
validate(){
local s=0
rclone lsd --config /root/.config/rclone/rclone.conf ${1}:/ &>/tmp/.SA_error.log.tmp && s=1
if [[ $s == 1 ]]; then
echo -e " [ OK ] ${1}\t Validation Successful!"
egrep -q ^${1}=. $gdsaDB || echo "${1}=0" >> $gdsaDB
else
echo -e " [WARN] ${1}\t Validation FAILURE!"
cat /tmp/.SA_error.log.tmp >> /tmp/SA_error.log
((gdsaFail++))
fi
}
i=0
numProcs=10
# parallelize validator for speeeeeed
for gdsa in $gdsaList; do
if (( i++ >= numProcs )); then
wait -n
fi
validate $gdsa &
sleep 0.1
done
wait
gdsaLeast=$(sort -gr -k2 -t'=' ${gdsaDB} | egrep ^GDSAC[0-9]+=. | tail -1 | cut -f1 -d'=')
[[ -n $gdsaFail ]] && echo -e " [WARN] $gdsaFail Failure(s). See /tmp/SA_error.log"
}
[[ $@ =~ --skip ]] || init_DB
############################################################################
# Least Usage Load Balancing of GDSA Accounts
############################################################################
numGdsa=$(cat $gdsaDB | wc -l)
maxDailyUpload=$(python3 -c "print(round($numGdsa * 750 / 1000, 3))")
echo -e " [INFO] START\tMax Concurrent Uploads: $maxConcurrentUploads, ${maxDailyUpload}TB Max Daily Upload"
echo -n '' > ${fileLock}
while true; do
# purge empty folders
find "${localDir}" -mindepth 2 -type d -empty -delete
# remote duplicates from fileLock
awk -i inplace '!seen[$0]++' ${fileLock}
# black magic: find list of all dirs that have files at least 1 minutes old that aren't hidden
# and put the deepest directories in an array, then sort by dirsize
sc=$(awk -F"/" '{print NF-2}' <<<${localDir})
unset a i
while IFS= read -r -u3 -d $'\0' dir; do
[[ $(find "${dir}" -type f -mmin -${modTime} -print -quit) == '' && ! $(find "${dir}" -name "*.partial~") ]] \
&& a[i++]=$(du -s "${dir}")
done 3< <(find ${localDir} -mindepth $sc -type d -not -path '*/\.*' -links 2 -not -empty -prune -print0)
# sort by largest files first
IFS=$'\n' uploadQueueBuffer=($(sort -gr <<<"${a[*]}"))
unset IFS
# iterate through each folder and upload
for i in $(seq 0 $((${#uploadQueueBuffer[@]}-1))); do
flag=0
# pause if max concurrent uploads limit is hit
numCurrentTransfers=$(grep -c "$localDir" $fileLock)
[[ $numCurrentTransfers -ge $maxConcurrentUploads ]] && break
# get least used gdsa account
gdsaLeast=$(sort -gr -k2 -t'=' ${gdsaDB} | egrep ^GDSAC[0-9]+=. | tail -1 | cut -f1 -d'=')
[[ -z $gdsaLeast ]] && echo -e " [FAIL] Failed To get gdsaLeast. Exiting." && exit 1
# upload folder (rclone_upload function will skip on filelocked folders)
if [[ -n "${uploadQueueBuffer[i]}" ]]; then
[[ -n $dbug ]] && echo -e " [DBUG] Supertransfer rclone_upload input: "${file}""
IFS=$'\t'
# |---uploadQueueBuffer--|
#input format: <dirsize> <upload_dir> <rclone> <remote_root_dir>
rclone_upload ${uploadQueueBuffer[i]} $gdsaLeast $remoteDir &
unset IFS
sleep 0.2
fi
done
unset -v uploadQueueBuffer[@]
sleep $sleepTime
done