move scripts to .bin

.bin/etc-files-list-update/util/find-new-urls.awk

@@ -0,0 +1,14 @@
+#!/usr/bin/awk -f
+BEGIN {
+  # load all the URLs we scanned already
+  command = "util/print-urls.sh"
+  while (command | getline) {
+    urls[$0] = 1 # add to set
+  }
+  close(command)
+  # get package URLs that do not appear in the list
+  command = "util/get-package-urls.sh"
+  while (command | getline) {
+    if (!($0 in urls)) print
+  }
+}

.bin/etc-files-list-update/util/get-package-urls.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# get new package URLs
+# load the list of amd64 packages from ubuntu
+export dist="$(cat current_distro)"
+export repo="http://archive.ubuntu.com/ubuntu"
+
+# print URLs
+curl $repo/dists/$dist/main/binary-amd64/Packages.gz | \
+  gzip -d | awk '/^Filename: / { print ENVIRON["repo"] "/" $2 }'

.bin/etc-files-list-update/util/print-urls.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# print every url in every file in deb-url-history directory
+for f in $(ls deb-url-history/)
+do
+  zcat "deb-url-history/$f"
+done

.bin/etc-files-list-update/util/scan-package.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+export url=$1
+
+tf=$(mktemp -d)
+wd=$(pwd)
+cd $tf
+wget "$url" -O output 2>/dev/null >/dev/null
+ar -x output # extracts data.tar.xz control.tar.xz
+
+# extract tar
+if [ -f control.tar.xz ]; then
+  xz -d control.tar.xz 2>/dev/null
+elif [ -f control.tar.zst ]; then # need to install zstd
+  zstd -d control.tar.zst 2>/dev/null
+elif [ -f control.tar.gz ]; then
+  tar -xzvf control.tar.gz 2>/dev/null >/dev/null
+else
+  (echo "$url unknown deb compression format" && ls) >> problems
+  exit 0
+fi
+
+# extract control
+tar -xvf control.tar 2>/dev/null >/dev/null
+
+# replace 2 spaces after md5sum with tab
+sed 's/^\([0-9a-zA-Z]*\)  /\1\t/' md5sums > inputdata
+
+# print filenames
+awk '
+BEGIN {
+  FS="\t"
+}
+{
+  gsub(/^\.\//,"",$2)
+  print "/" $2
+}
+' inputdata
+
+# cleanup
+cd "$wd"
+rm -rf $tf