From a94de2d3def01fb20b8abba77867edf1b7e0e48d Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Thu, 3 Aug 2023 12:51:09 -0700
Subject: [PATCH 1/7] wip

---
 examples/deb/BUILD.bazel                   |  3 +++
 oci/private/image.bzl                      |  6 +++--
 oci/private/image.sh.tpl                   | 31 ++++++++++++++++++----
 oci/private/registry/crane_launcher.sh.tpl | 12 ++++-----
 oci/private/registry/zot_launcher.sh.tpl   | 15 +++++++----
 5 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/examples/deb/BUILD.bazel b/examples/deb/BUILD.bazel
index 8cd7826f..15481a8b 100644
--- a/examples/deb/BUILD.bazel
+++ b/examples/deb/BUILD.bazel
@@ -12,6 +12,7 @@ _ARCH = [
         name = "decompress_" + architecture,
         srcs = ["@bash_{}//:layer".format(architecture)],
         outs = ["_{}.tar".format(architecture)],
+        tags = ["manual"],
         cmd = "xz --decompress --stdout $< >$@",
     )
     for architecture in _ARCH
@@ -22,6 +23,7 @@ _ARCH = [
         name = "image_" + architecture,
         architecture = architecture,
         os = "linux",
+        tags = ["manual"],
         tars = ["_{}.tar".format(architecture)],
     )
     for architecture in _ARCH
@@ -30,5 +32,6 @@ _ARCH = [
 oci_tarball(
     name = "tarball",
     image = ":image_amd64",
+    tags = ["manual"],
     repo_tags = ["test:test"],
 )
diff --git a/oci/private/image.bzl b/oci/private/image.bzl
index 56ba5894..ab7aab51 100644
--- a/oci/private/image.bzl
+++ b/oci/private/image.bzl
@@ -108,6 +108,8 @@ def _oci_image_impl(ctx):
 
     launcher = ctx.actions.declare_file("image_%s.sh" % ctx.label.name)
 
+    output = ctx.actions.declare_directory(ctx.label.name)
+
     ctx.actions.expand_template(
         template = ctx.file._image_sh_tpl,
         output = launcher,
@@ -116,7 +118,7 @@ def _oci_image_impl(ctx):
             "{{registry_launcher_path}}": registry.registry_info.launcher.path,
             "{{crane_path}}": crane.crane_info.binary.path,
             "{{jq_path}}": jq.jqinfo.bin.path,
-            "{{storage_dir}}": "/".join([ctx.bin_dir.path, ctx.label.package, "storage_%s" % ctx.label.name]),
+            "{{storage_dir}}": output.path,
             "{{empty_tar}}": ctx.file._empty_tar.path,
         },
     )
@@ -174,7 +176,7 @@ def _oci_image_impl(ctx):
         args.add(ctx.file.annotations.path, format = "--annotations-file=%s")
         inputs_depsets.append(depset([ctx.file.annotations]))
 
-    output = ctx.actions.declare_directory(ctx.label.name)
+
     args.add(output.path, format = "--output=%s")
 
     action_env = {}
diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl
index b41ac978..505f8cd2 100644
--- a/oci/private/image.sh.tpl
+++ b/oci/private/image.sh.tpl
@@ -6,7 +6,7 @@ set -o pipefail -o errexit -o nounset
 # NB: --output argument is an option only understood by this wrapper and will pull artifact image into a oci layout.
 
 readonly REGISTRY_LAUNCHER="{{registry_launcher_path}}"
-readonly CRANE="{{crane_path}}"
+readonly CRANE="/Users/thesayyn/Documents/go-containerregistry/main"
 readonly JQ="{{jq_path}}"
 readonly STORAGE_DIR="{{storage_dir}}"
 
@@ -33,7 +33,7 @@ function get_option() {
 
 function empty_base() {
     local registry=$1
-    local ref="$registry/oci/empty_base:latest"
+    local ref="$registry/image:latest"
     ref="$("${CRANE}" append --oci-empty-base -t "${ref}" -f {{empty_tar}})"
     ref=$("${CRANE}" config "${ref}" | "${JQ}"  ".rootfs.diff_ids = [] | .history = []" | "${CRANE}" edit config "${ref}")
     ref=$("${CRANE}" manifest "${ref}" | "${JQ}"  ".layers = []" | "${CRANE}" edit manifest "${ref}")
@@ -58,7 +58,7 @@ function base_from_layout() {
     local oci_layout_path=$1
     local registry=$2
 
-    "${CRANE}" push "${oci_layout_path}" "${registry}/oci/layout:latest" --image-refs "${refs}" > "${output}" 2>&1
+    "${CRANE}" push "${oci_layout_path}" "${registry}/image:latest" --image-refs "${refs}" > "${output}" 2>&1
 
     echo "${output}" >&2
 
@@ -79,6 +79,23 @@ EOF
     cat "${refs}"
 }
 
+# removes unreferenced blobs from oci-layout
+function gc() {
+    local ref="$1"
+    local digest=$("${CRANE}" digest "${ref}")
+    local blobs=($("${CRANE}" manifest "${ref}" | "${JQ}" -r --arg digest "$digest" '([.layers[].digest] + [.config.digest, $digest]) | flatten | .[]'))
+    for blob_dir in ${STORAGE_DIR}/blobs/* ; do
+        local algo="$(basename ${blob_dir})"
+        for blob_path in ${blob_dir}/* ; do
+            local blob_digest="$(basename ${blob_path})"
+            local hash="${algo}:${blob_digest}"
+            if ! [[ "${blobs[@]}" =~ "$hash" ]]; then
+                rm $blob_path
+            fi
+        done
+    done
+}
+
 # this will redirect stderr(2) to stderr file.
 {
 source "${REGISTRY_LAUNCHER}" 
@@ -160,7 +177,11 @@ if [ ${#ENV_EXPANSIONS[@]} -ne 0 ]; then
 fi
 
 if [ -n "$OUTPUT" ]; then
-    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
+    # "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
+    # gc "${REF}"
+    stop_registry "${STORAGE_DIR}"
+    cat $STDERR
 fi
 
-} 2>> "${STDERR}"
+} 
+# 2>> "${STDERR}"
diff --git a/oci/private/registry/crane_launcher.sh.tpl b/oci/private/registry/crane_launcher.sh.tpl
index c823bdf6..787d1646 100644
--- a/oci/private/registry/crane_launcher.sh.tpl
+++ b/oci/private/registry/crane_launcher.sh.tpl
@@ -1,16 +1,12 @@
 readonly SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
-readonly CRANE_REGISTRY_BIN="${SCRIPT_DIR}/{{CRANE}}"
+readonly CRANE_REGISTRY_BIN="/Users/thesayyn/Documents/go-containerregistry/main"
 
 function start_registry() {
     local storage_dir="$1"
     local output="$2"
     local deadline="${3:-5}"
 
-    mkdir -p "${storage_dir}"
-    # --blobs-to-disk uses go's os.TempDir() function which is equal to TMPDIR under *nix.
-    # https://pkg.go.dev/os#TempDir
-    TMPDIR="${storage_dir}" TMP="${storage_dir}" \
-    "${CRANE_REGISTRY_BIN}" registry serve --blobs-to-disk >> $output 2>&1 &
+    "${CRANE_REGISTRY_BIN}" registry serve --disk="${storage_dir}" >> $output 2>&1 &
 
     local timeout=$((SECONDS+${deadline}))
 
@@ -26,4 +22,8 @@ function start_registry() {
     fi
     echo "127.0.0.1:${port}"
     return 0
+}
+
+function stop_registry() { 
+    :
 }
\ No newline at end of file
diff --git a/oci/private/registry/zot_launcher.sh.tpl b/oci/private/registry/zot_launcher.sh.tpl
index 2ad95d95..e344e07b 100644
--- a/oci/private/registry/zot_launcher.sh.tpl
+++ b/oci/private/registry/zot_launcher.sh.tpl
@@ -5,14 +5,14 @@ function start_registry() {
     local storage_dir="$1"
     local output="$2"
     local deadline="${3:-5}"
-    local config_path="$1/config.json"
+    local config_path="$storage_dir/config.json"
 
-    mkdir -p "${storage_dir}"
+    echo "$storage_dir" >&2
     cat > "${config_path}" <<EOF
 {
-    "storage": {"rootDirectory": "$1" },
-    "http": { "port": "0", "address": "127.0.0.1" },
-    "log": { "level": "info" }
+    "storage":{ "rootDirectory": "$storage_dir/all",  "dedupe": false, "subPaths": {"/image": {"rootDirectory": "$storage_dir"}}},
+    "http":{ "port": "0", "address": "127.0.0.1" },
+    "log":{ "level": "debug" }
 }
 EOF
     HOME="${TMPDIR}" "${ZOT}" serve "${config_path}" >> $output 2>&1 &
@@ -31,4 +31,9 @@ EOF
     fi
     echo "127.0.0.1:${port}"
     return 0
+}
+
+function stop_registry() {
+    local storage_dir="$1"
+    rm "$storage_dir/config.json"
 }
\ No newline at end of file

From 43ed02b99db158d5968a9c4a882200b9d2844a1e Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Fri, 4 Aug 2023 15:38:21 -0700
Subject: [PATCH 2/7] fix zot

---
 oci/private/image.sh.tpl                 | 16 ++++++++--------
 oci/private/registry/zot_launcher.sh.tpl |  9 ++++++---
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl
index 505f8cd2..2480dddd 100644
--- a/oci/private/image.sh.tpl
+++ b/oci/private/image.sh.tpl
@@ -60,8 +60,6 @@ function base_from_layout() {
 
     "${CRANE}" push "${oci_layout_path}" "${registry}/image:latest" --image-refs "${refs}" > "${output}" 2>&1
 
-    echo "${output}" >&2
-
     if grep -q "MANIFEST_INVALID" "${output}"; then
     cat >&2 << EOF
 
@@ -79,7 +77,7 @@ EOF
     cat "${refs}"
 }
 
-# removes unreferenced blobs from oci-layout
+# removes unreferenced blobs from oci-layout and normalizes index.json
 function gc() {
     local ref="$1"
     local digest=$("${CRANE}" digest "${ref}")
@@ -94,6 +92,9 @@ function gc() {
             fi
         done
     done
+    mv "${STORAGE_DIR}/index.json" "${STORAGE_DIR}/temp.json"
+    "${JQ}" --arg digest "$digest" '.manifests |= [map(select(.digest == $digest and .annotations == null))[0]]' "${STORAGE_DIR}/temp.json" > "${STORAGE_DIR}/index.json"
+    rm "${STORAGE_DIR}/temp.json"
 }
 
 # this will redirect stderr(2) to stderr file.
@@ -177,11 +178,10 @@ if [ ${#ENV_EXPANSIONS[@]} -ne 0 ]; then
 fi
 
 if [ -n "$OUTPUT" ]; then
-    # "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
-    # gc "${REF}"
+
+    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
+    gc "${REF}"
     stop_registry "${STORAGE_DIR}"
-    cat $STDERR
 fi
 
-} 
-# 2>> "${STDERR}"
+} 2>> "${STDERR}"
diff --git a/oci/private/registry/zot_launcher.sh.tpl b/oci/private/registry/zot_launcher.sh.tpl
index e344e07b..4a8291ae 100644
--- a/oci/private/registry/zot_launcher.sh.tpl
+++ b/oci/private/registry/zot_launcher.sh.tpl
@@ -10,9 +10,9 @@ function start_registry() {
     echo "$storage_dir" >&2
     cat > "${config_path}" <<EOF
 {
-    "storage":{ "rootDirectory": "$storage_dir/all",  "dedupe": false, "subPaths": {"/image": {"rootDirectory": "$storage_dir"}}},
+    "storage": { "rootDirectory": "$storage_dir/..", "dedupe": false, "commit": true },
     "http":{ "port": "0", "address": "127.0.0.1" },
-    "log":{ "level": "debug" }
+    "log":{ "level": "info" }
 }
 EOF
     HOME="${TMPDIR}" "${ZOT}" serve "${config_path}" >> $output 2>&1 &
@@ -33,7 +33,10 @@ EOF
     return 0
 }
 
+
+
 function stop_registry() {
     local storage_dir="$1"
-    rm "$storage_dir/config.json"
+    rm -r "${storage_dir}/.uploads"
+    rm -r "${storage_dir}/config.json"
 }
\ No newline at end of file

From c59f2565cb16ba48114bde5179d23a294c92be16 Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Fri, 4 Aug 2023 15:40:50 -0700
Subject: [PATCH 3/7] revert

---
 examples/deb/BUILD.bazel | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/examples/deb/BUILD.bazel b/examples/deb/BUILD.bazel
index 15481a8b..8cd7826f 100644
--- a/examples/deb/BUILD.bazel
+++ b/examples/deb/BUILD.bazel
@@ -12,7 +12,6 @@ _ARCH = [
         name = "decompress_" + architecture,
         srcs = ["@bash_{}//:layer".format(architecture)],
         outs = ["_{}.tar".format(architecture)],
-        tags = ["manual"],
         cmd = "xz --decompress --stdout $< >$@",
     )
     for architecture in _ARCH
@@ -23,7 +22,6 @@ _ARCH = [
         name = "image_" + architecture,
         architecture = architecture,
         os = "linux",
-        tags = ["manual"],
         tars = ["_{}.tar".format(architecture)],
     )
     for architecture in _ARCH
@@ -32,6 +30,5 @@ _ARCH = [
 oci_tarball(
     name = "tarball",
     image = ":image_amd64",
-    tags = ["manual"],
     repo_tags = ["test:test"],
 )

From 125c19c31e325ea0b596f667c6f60c7dc34ec712 Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Wed, 11 Oct 2023 17:40:30 -0400
Subject: [PATCH 4/7] use prune

---
 oci/private/image.sh.tpl                 | 24 +-----------------------
 oci/private/registry/zot_launcher.sh.tpl |  2 +-
 2 files changed, 2 insertions(+), 24 deletions(-)

diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl
index 2480dddd..92ec06fb 100644
--- a/oci/private/image.sh.tpl
+++ b/oci/private/image.sh.tpl
@@ -77,26 +77,6 @@ EOF
     cat "${refs}"
 }
 
-# removes unreferenced blobs from oci-layout and normalizes index.json
-function gc() {
-    local ref="$1"
-    local digest=$("${CRANE}" digest "${ref}")
-    local blobs=($("${CRANE}" manifest "${ref}" | "${JQ}" -r --arg digest "$digest" '([.layers[].digest] + [.config.digest, $digest]) | flatten | .[]'))
-    for blob_dir in ${STORAGE_DIR}/blobs/* ; do
-        local algo="$(basename ${blob_dir})"
-        for blob_path in ${blob_dir}/* ; do
-            local blob_digest="$(basename ${blob_path})"
-            local hash="${algo}:${blob_digest}"
-            if ! [[ "${blobs[@]}" =~ "$hash" ]]; then
-                rm $blob_path
-            fi
-        done
-    done
-    mv "${STORAGE_DIR}/index.json" "${STORAGE_DIR}/temp.json"
-    "${JQ}" --arg digest "$digest" '.manifests |= [map(select(.digest == $digest and .annotations == null))[0]]' "${STORAGE_DIR}/temp.json" > "${STORAGE_DIR}/index.json"
-    rm "${STORAGE_DIR}/temp.json"
-}
-
 # this will redirect stderr(2) to stderr file.
 {
 source "${REGISTRY_LAUNCHER}" 
@@ -178,9 +158,7 @@ if [ ${#ENV_EXPANSIONS[@]} -ne 0 ]; then
 fi
 
 if [ -n "$OUTPUT" ]; then
-
-    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
-    gc "${REF}"
+    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci --prune
     stop_registry "${STORAGE_DIR}"
 fi
 
diff --git a/oci/private/registry/zot_launcher.sh.tpl b/oci/private/registry/zot_launcher.sh.tpl
index 4a8291ae..dd2053e2 100644
--- a/oci/private/registry/zot_launcher.sh.tpl
+++ b/oci/private/registry/zot_launcher.sh.tpl
@@ -37,6 +37,6 @@ EOF
 
 function stop_registry() {
     local storage_dir="$1"
-    rm -r "${storage_dir}/.uploads"
+    rm -rf "${storage_dir}/.uploads"
     rm -r "${storage_dir}/config.json"
 }
\ No newline at end of file

From c2bd09350d45ecc2c05f04614a7b1cdea07925ef Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Wed, 29 Nov 2023 15:44:59 -0800
Subject: [PATCH 5/7] cleanup

---
 oci/private/image.sh.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl
index 92ec06fb..d7e46618 100644
--- a/oci/private/image.sh.tpl
+++ b/oci/private/image.sh.tpl
@@ -6,7 +6,7 @@ set -o pipefail -o errexit -o nounset
 # NB: --output argument is an option only understood by this wrapper and will pull artifact image into a oci layout.
 
 readonly REGISTRY_LAUNCHER="{{registry_launcher_path}}"
-readonly CRANE="/Users/thesayyn/Documents/go-containerregistry/main"
+readonly CRANE="{{crane_path}}"
 readonly JQ="{{jq_path}}"
 readonly STORAGE_DIR="{{storage_dir}}"
 

From 9d105dc379bc78cfb15c1590e753c158d2261a06 Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Wed, 29 Nov 2023 16:26:34 -0800
Subject: [PATCH 6/7] make it work

---
 e2e/custom_registry/registry/launcher.sh   |  4 ++++
 oci/private/image.sh.tpl                   | 10 +++++++---
 oci/private/registry/crane_launcher.sh.tpl |  2 +-
 oci/private/registry/zot_launcher.sh.tpl   |  3 ---
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/e2e/custom_registry/registry/launcher.sh b/e2e/custom_registry/registry/launcher.sh
index 85a617f6..d3eb2ec0 100755
--- a/e2e/custom_registry/registry/launcher.sh
+++ b/e2e/custom_registry/registry/launcher.sh
@@ -21,4 +21,8 @@ function start_registry() {
     fi
     echo "127.0.0.1:${port}"
     return 0
+}
+
+function stop_registry() {
+    :
 }
\ No newline at end of file
diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl
index d7e46618..e34523c0 100644
--- a/oci/private/image.sh.tpl
+++ b/oci/private/image.sh.tpl
@@ -33,7 +33,7 @@ function get_option() {
 
 function empty_base() {
     local registry=$1
-    local ref="$registry/image:latest"
+    local ref="$registry/oci/empty_base:latest"
     ref="$("${CRANE}" append --oci-empty-base -t "${ref}" -f {{empty_tar}})"
     ref=$("${CRANE}" config "${ref}" | "${JQ}"  ".rootfs.diff_ids = [] | .history = []" | "${CRANE}" edit config "${ref}")
     ref=$("${CRANE}" manifest "${ref}" | "${JQ}"  ".layers = []" | "${CRANE}" edit manifest "${ref}")
@@ -158,8 +158,12 @@ if [ ${#ENV_EXPANSIONS[@]} -ne 0 ]; then
 fi
 
 if [ -n "$OUTPUT" ]; then
-    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci --prune
-    stop_registry "${STORAGE_DIR}"
+    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci --annotate-ref
+    mv "${OUTPUT}/index.json" "${OUTPUT}/temp.json"
+    "${JQ}" --arg ref "${REF}" '.manifests |= map(select(.annotations["org.opencontainers.image.ref.name"] == $ref)) | del(.manifests[0].annotations)' "${OUTPUT}/temp.json" >  "${OUTPUT}/index.json"
+    rm "${OUTPUT}/temp.json"
+    "${CRANE}" layout gc "./${OUTPUT}"
+    stop_registry "${OUTPUT}"
 fi
 
 } 2>> "${STDERR}"
diff --git a/oci/private/registry/crane_launcher.sh.tpl b/oci/private/registry/crane_launcher.sh.tpl
index 787d1646..b1bf5e50 100644
--- a/oci/private/registry/crane_launcher.sh.tpl
+++ b/oci/private/registry/crane_launcher.sh.tpl
@@ -1,5 +1,5 @@
 readonly SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
-readonly CRANE_REGISTRY_BIN="/Users/thesayyn/Documents/go-containerregistry/main"
+readonly CRANE_REGISTRY_BIN="${SCRIPT_DIR}/{{CRANE}}"
 
 function start_registry() {
     local storage_dir="$1"
diff --git a/oci/private/registry/zot_launcher.sh.tpl b/oci/private/registry/zot_launcher.sh.tpl
index dd2053e2..917f6930 100644
--- a/oci/private/registry/zot_launcher.sh.tpl
+++ b/oci/private/registry/zot_launcher.sh.tpl
@@ -7,7 +7,6 @@ function start_registry() {
     local deadline="${3:-5}"
     local config_path="$storage_dir/config.json"
 
-    echo "$storage_dir" >&2
     cat > "${config_path}" <<EOF
 {
     "storage": { "rootDirectory": "$storage_dir/..", "dedupe": false, "commit": true },
@@ -33,8 +32,6 @@ EOF
     return 0
 }
 
-
-
 function stop_registry() {
     local storage_dir="$1"
     rm -rf "${storage_dir}/.uploads"

From e53553cf489ce2fc455c361156c3311e602796d0 Mon Sep 17 00:00:00 2001
From: thesayyn <thesayyn@gmail.com>
Date: Thu, 11 Jan 2024 12:26:43 -0800
Subject: [PATCH 7/7] fix

---
 oci/private/image.sh.tpl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl
index b3294ee0..ee7f6e30 100644
--- a/oci/private/image.sh.tpl
+++ b/oci/private/image.sh.tpl
@@ -164,7 +164,6 @@ if [ -n "$OUTPUT" ]; then
     "${JQ}" --arg ref "${REF}" '.manifests |= map(select(.annotations["org.opencontainers.image.ref.name"] == $ref)) | del(.manifests[0].annotations)' "${OUTPUT}/temp.json" >  "${OUTPUT}/index.json"
     rm "${OUTPUT}/temp.json"
     "${CRANE}" layout gc "./${OUTPUT}"
-    stop_registry "${OUTPUT}"
 fi
 
 } 2>> "${STDERR}"