Initial testing

.gitignore

@@ -0,0 +1 @@
+notes.secret.txt

Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:latest
+LABEL maintainer="fred.damstra@gmail.com"
+LABEL version="0.1"
+LABEL description="Docker image that sorts my midjourney albums"
+
+RUN apk add --no-cache bash bash-completion
+
+COPY . /opt/midjourney/
+RUN chmod 755 /opt/midjourney/sort_and_upload.sh \
+ && chmod 755 /opt/midjourney/bin/gdrive \
+ && mkdir /scratch \
+ && mkdir /shared
+
+CMD ["/opt/midjourney/sort_and_upload.sh"]

README.md

@@ -0,0 +1,15 @@
+# midjourney sort and upload
+
+Sorts images by size and synchronized them to google drive, apparently.
+
+## GDRIVE Credentials
+
+I don't know where I got these, but it needs them.
+
+```
+kubectl create secret generic midjourney-sort-and-upload \
+  --from-literal=SMALL=longstringofnumbers \
+  --from-literal=MEDIUM=longstringofnumbers \
+  --from-literal=LARGE=longstringofnumbers
+```
+

k8s.midjourney-sort-and-upload.yaml

@@ -0,0 +1,118 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: midjourney-sort-and-upload
+spec:
+  capacity:
+    storage: 10Ti
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain # Keep 4eva
+  storageClassName: default
+  mountOptions:
+    - hard
+    - nfsvers=3
+  nfs:
+    path: /mnt/DroboFS/Shares/Kubernetes/volumes/static/midjourney-sort-and-upload
+    server: 10.42.42.10
+  claimRef:
+    name: midjourney-sort-and-upload
+    namespace: default
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: midjourney-sort-and-upload
+spec:
+  storageClassName: standard
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Ti
+status: {}
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: midjourney-sort-and-upload-workdir
+spec:
+  capacity:
+    storage: 10Ti
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain # Keep 4eva
+  storageClassName: default
+  mountOptions:
+    - hard
+    - nfsvers=3
+  nfs:
+    path: /mnt/DroboFS/Shares/Public/midjourney
+    server: 10.42.42.10
+  claimRef:
+    name: midjourney-sort-and-upload-workdir
+    namespace: default
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: midjourney-sort-and-upload-workdir
+spec:
+  storageClassName: standard
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Ti
+status: {}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cron-midjourney-sort-and-upload
+spec:
+  schedule: "0 */1 * * *"
+  concurrencyPolicy: Forbid
+  # Start up to 30 minutes late
+  startingDeadlineSeconds: 1800
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 1700
+      backoffLimit: 4
+      template:
+        spec:
+          containers:
+          - name: cron-midjourney-sort-and-upload
+            image: fdamstra/midjourney-sort-and-upload
+            envFrom:
+              - secretRef:
+                  name: midjourney-sort-and-upload
+            volumeMounts:
+            - name: midjourney-sort-and-upload
+              mountPath: /scratch/
+            - name: midjourney-sort-and-upload-workdir
+              mountPath: /midjourney/
+          dnsPolicy: "None"
+          dnsConfig:
+            nameservers:
+              - 10.42.42.239
+              - 10.42.42.1
+            searches:
+              - default.svc.cluster.local
+              - svc.cluster.local
+              - cluster.local
+            options:
+              - name: ndots
+                value: "2"
+              - name: edns0
+              - name: trust-ad
+          restartPolicy: Never
+          volumes:
+          - name: midjourney-sort-and-upload
+            persistentVolumeClaim:
+              claimName: midjourney-sort-and-upload
+          - name: midjourney-sort-and-upload-workdir
+            persistentVolumeClaim:
+              claimName: midjourney-sort-and-upload-workdir

kaniko-midjourney-sort-and-upload.yaml

@@ -0,0 +1,62 @@
+---
+# Kaniko builds images without docker
+#
+# Use 'kubectl create -f kaniko.yaml' to execute
+# 
+# NOTES:
+# If this is your first kaniko, you need to add a token:
+# In gogs: User->Settings->Applications
+# kubectl create secret generic git-token --from-literal='GIT_TOKEN=<your-token>'
+# 
+# You need to add credentials to the docker registry:
+# kubectl create secret docker-registry  docker-regcred \
+#  --docker-server=https://index.docker.io/v1/ \
+#  --docker-username=<your-username> \
+#  --docker-password=<your-password>
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kaniko-midjourney-sort-and-upload
+spec:
+  ttlSecondsAfterFinished: 100
+  backoffLimit: 4
+  template:
+    spec:
+      containers:
+      - name: kaniko-midjourney-sort-and-upload
+        image: gcr.io/kaniko-project/executor:v1.9.1 
+        args:
+        - "--dockerfile=./Dockerfile"
+        - "--context=git://git.monkeybox.org/Containers/midjourney-sort-and-upload#refs/heads/main"
+        - "--destination=fdamstra/midjourney-sort-and-upload:latest"
+        env:
+          - name: GIT_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: git-token
+                key: GIT_TOKEN
+        volumeMounts:
+        - name: docker-config
+          mountPath: /kaniko/.docker/
+      dnsPolicy: "None"
+      dnsConfig:
+        nameservers:
+          - 10.42.42.239
+          - 10.42.42.1
+        searches:
+          - default.svc.cluster.local
+          - svc.cluster.local
+          - cluster.local
+        options:
+          - name: ndots
+            value: "2"
+          - name: edns0
+          - name: trust-ad
+      restartPolicy: Never
+      volumes:
+      - name: docker-config
+        secret:
+          secretName: docker-regcred
+          items:
+              - key: .dockerconfigjson
+                path: config.json

sort_and_upload.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+
+GDRIVE=/opt/midjourney/gdrive
+LOGFILE=/scratch/midjourney-sort-and-upload.log
+MIDJOURNEYDIR=/midjourney/
+
+echo ""
+echo "Starting midjourney sync and upload - $(date)" 2>&1 | tee -a ${LOGFILE}
+
+if [[ ! -v ${SMALL} ]]; then
+  echo "ERROR: No gdrive key for SMALL" 2>&1 | tee -a ${LOGFILE}
+  exit -1
+fi
+
+if [[ ! -v ${MEDIUM} ]]; then
+  echo "ERROR: No gdrive key for MEDIUM" 2>&1 | tee -a ${LOGFILE}
+  exit -1
+fi
+
+if [[ ! -v ${LARGE} ]]; then
+  echo "ERROR: No gdrive key for LARGE" 2>&1 | tee -a ${LOGFILE}
+  exit -1
+fi
+
+SAVEIFS=$IFS
+IFS=$(echo -en "\n\b")
+
+if compgen -G "/shared/midjourney/images/*.png" > /dev/null; then
+  echo New files. Sorting. 2>&1 | tee -a ${LOGFILE}
+  for i in ${MIDJOURNEYDIR}/images/*.png; do
+    maxdim=$(file "$i" | awk '{ print ($6 > $8)? $6 : $8 }' | sed 's/,$//')
+    if [ $maxdim -lt 1024 ]; then
+      echo SMALL: "$i" 2>&1 | tee -a ${LOGFILE}
+      rsync $i ${MIDJOURNEYDIR}/images.small/ 2>&1 | tee -a ${LOGFILE}
+      if [ $? -eq 0 ]; then
+        mv $i ${MIDJOURNEYDIR}/images.filtered/ 2>&1 | tee -a ${LOGFILE}
+      fi
+    elif [ $maxdim -lt 1281 ]; then
+      echo MEDIUM: "$i" 2>&1 | tee -a ${LOGFILE}
+      rsync $i ${MIDJOURNEYDIR}/images.medium/ 2>&1 | tee -a ${LOGFILE}
+      if [ $? -eq 0 ]; then
+        mv $i ${MIDJOURNEYDIR}/images.filtered/ 2>&1 | tee -a ${LOGFILE}
+      fi
+    else
+      echo LARGE: "$i" 2>&1 | tee -a ${LOGFILE}
+      rsync $i ${MIDJOURNEYDIR}/images.large/ 2>&1 | tee -a ${LOGFILE}
+      if [ $? -eq 0 ]; then
+        mv $i ${MIDJOURNEYDIR}/images.filtered/ 2>&1 | tee -a ${LOGFILE}
+      fi
+    fi
+  done
+else
+  echo No new files. No need to sort. 2>&1 | tee -a ${LOGFILE}
+fi
+
+echo Syncing Small 2>&1 | tee -a ${LOGFILE}
+$GDRIVE sync upload --keep-largest /shared/midjourney/images.small/ $SMALL 2>&1 | tee -a ${LOGFILE}
+echo Syncing Medium 2>&1 | tee -a ${LOGFILE}
+$GDRIVE sync upload --keep-largest /shared/midjourney/images.medium/ $MEDIUM 2>&1 | tee -a ${LOGFILE}
+echo Syncing Large 2>&1 | tee -a ${LOGFILE}
+$GDRIVE sync upload --keep-largest /shared/midjourney/images.large/ $LARGE 2>&1 | tee -a ${LOGFILE}