diff options
| -rwxr-xr-x | apkcontent2sqlite.sh | 139 | ||||
| -rwxr-xr-x | apkindex2sqlite.sh | 136 |
2 files changed, 160 insertions, 115 deletions
diff --git a/apkcontent2sqlite.sh b/apkcontent2sqlite.sh index f7a381c..21d4284 100755 --- a/apkcontent2sqlite.sh +++ b/apkcontent2sqlite.sh @@ -1,48 +1,99 @@ #!/bin/sh -base="/var/www/localhost/htdocs/alpine" -#release=$(readlink $base/latest-stable) -release="edge" +packages="repository" +filelist="filelist" repos="main testing" archs="x86 x86_64 armhf" -output="output.csv" -database="filelist.db" - -create_filelist() { - local apk="$1" repo="$2" arch="$3" - local filelist=$(tar --exclude ".*" -ztf "$apk") - local filename="${apk##*/}" - local pkgname="${filename%-*-*}" - IFS=$'\n' - for item in $filelist; do - file="${item##*/}" - path="${item%/*}" - case "$file" in - *,*) file="\"$file\"" ;; - esac - case "$path" in - *,*) path="\"$path\"" ;; - esac - [ ! -z "$file" ] && \ - echo "$file,$path,$pkgname,$repo,$arch" >> $output - done - unset IFS -} - - -rm -f $output $database - -printf "CREATE TABLE filelist(file text, path text, pkgname text, repo text, arch text);" | sqlite3 $database - -for repo in $repos; do - for arch in $archs; do - printf "Starting: $repo/$arch" - for apk in $base/$release/$repo/$arch/*.apk; do - printf . - create_filelist "$apk" "$repo" "$arch" - done - printf "\n" - done -done - -printf ".mode csv\n.import $output filelist" | sqlite3 $database +db="db/filelist.db" +csv="csv/filelist.csv" +turbo_db="/var/www/aports-turbo/db/apkindex.db" + +rm_stale_filelist() { + for repo in $repos; do + for arch in $archs; do + for list in $filelist/$repo/$arch/*.csv; do + local file="${list##*/}" + local apk="${file%.*}" + if [ ! -e "$packages/$repo/$arch/$apk" ]; then + rm -f "$list" + result=1 + fi + done + done + done +} + +update_filelist() { + for repo in $repos; do + for arch in $archs; do + mkdir -p "$filelist/$repo/$arch" + for apk in $packages/$repo/$arch/*.apk; do + local file="${apk##*/}" + if [ ! -e "$filelist/$repo/$arch/${file}.csv" ]; then + create_apk_csv "$apk" "$repo" "$arch" >> \ + "$filelist/$repo/$arch/${file}.csv" + result=1 + fi + done + done + done +} + +apk_sync() { + for repo in $repos; do + rsync -a --delete-before $rsync/$repo $packages || return 1 + done +} + +create_apk_csv() { + local apk="$1" repo="$2" arch="$3" + local list=$(tar --exclude ".*" -ztf "$apk") + local filename="${apk##*/}" + local pkgname="${filename%-*-*}" + IFS=$'\n' + for line in $list; do + local file="${line##*/}" + local path="${line%/*}" + case "$file" in + *,*) file="\"$file\"" ;; + esac + case "$path" in + *,*) path="\"$path\"" ;; + esac + [ ! -z "$file" ] && \ + echo "$file,$path,$pkgname,$repo,$arch" + done + unset IFS +} + +cat_csv() { + rm -f $csv + for repo in $repos; do + for arch in $archs; do + cat $filelist/$repo/$arch/*.csv >> $csv + done + done +} + +prepare() { + rm -f $db $csv + mkdir -p ${csv%/*} ${db%/*} +} + +create_db() { + printf "CREATE TABLE filelist(file text, path text, pkgname text, repo text, arch text);" | sqlite3 $db + printf ".mode csv\n.import $csv filelist" | sqlite3 $db + printf "CREATE INDEX file on filelist (file);" | sqlite3 $db + printf "CREATE INDEX arch on filelist (arch);" | sqlite3 $db + printf "CREATE INDEX pkgname on filelist (pkgname);" | sqlite3 $db +} + +mv_db() { + mv $db $turbo_db +} + +# just do it +prepare && apk_sync && update_filelist && rm_stale_filelist +# only update db when we have changes +[ ! -z "$result" ] && cat_csv && create_db && mv_db + diff --git a/apkindex2sqlite.sh b/apkindex2sqlite.sh index d2c96f4..64e3919 100755 --- a/apkindex2sqlite.sh +++ b/apkindex2sqlite.sh @@ -1,89 +1,83 @@ #!/bin/sh -apkindex="APKINDEX" -csvfile="output.csv" -database="apkindex.db" - -rm -f $csvfile $database +packages="repository" +repos="main testing" +archs="x86 x86_64 armhf" +db="db/apkindex.db" +csv="csv/apkindex.csv" +turbo_db="/var/www/aports-turbo/db/apkindex.db" read_line() { local type=${1%%:*} local var=${1#*:} case "$type" in - P) #name - name="$var" - ;; - V) #version - version="$var" - ;; - T) #description - desc="\"$var\"" - ;; - U) #url - url="\"$var\"" - ;; - L) #license - lic="$var" - ;; - A) #arch - arch="$var" - ;; - D) #depends - deps="$var" - ;; - C) #csum - csum="$var" - ;; - S) #size - size="$var" - ;; - I) #installed_size - install_size="$var" - ;; - p) #provides - provides="$var" - ;; - i) #install_if - install_if="$var" - ;; - o) #origin - origin="$var" - ;; - m) #maintainer - maintainer="$var" - ;; - t) #build_time - build_time="$var" - ;; - c) #commit - commit="$var" - ;; - F|M|R|Z|r|q|a|s|f) #installed db entries - return 1 - ;; - esac + P) name="$var" ;; + V) version="$var" ;; + T) desc="\"$var\"" ;; + U) url="\"$var\"" ;; + L) lic="$var" ;; + A) parch="$var" ;; + D) deps="$var" ;; + C) csum="$var" ;; + S) size="$var" ;; + I) install_size="$var" ;; + p) provides="$var" ;; + i) install_if="$var" ;; + o) origin="$var" ;; + m) maintainer="$var" ;; + t) build_time="$var" ;; + c) commit="$var" ;; + esac } reset_vars() { - name= version= desc= url= lic= arch= deps= csum= size= install_size= + name= version= desc= url= lic= parch= deps= csum= size= install_size= provides= install_if= origin= maintainer= build_time= commit= } write_line() { - echo "$name,$version,$desc,$url,$lic,$arch,$deps,$csum,$size,$install_size,$provides,$install_if,$origin,$maintainer,$build_time,$commit" >> \ - $csvfile + echo "$name,$version,$desc,$url,$lic,$parch,$deps,$csum,$size,$install_size,$provides,$install_if,$origin,$maintainer,$build_time,$commit,$repo" +} + +init_db() { + rm -f $db + printf "CREATE TABLE apkindex('name' text, 'version' text, 'desc' text, + 'url' text, 'lic' text, 'arch' text, 'deps' text, 'csum' text, + 'size' text, 'install_size' text, 'provides' text, 'install_if' text, + 'origin' text, 'maintainer' text, 'build_time' integer, 'commit' text, + 'repo' text);" | sqlite3 $db +} + +write_db() { + printf ".mode csv\n.import $csv apkindex" | sqlite3 $db + printf "CREATE INDEX name on apkindex (name);" | sqlite3 $db + printf "CREATE INDEX arch on apkindex (arch);" | sqlite3 $db + printf "CREATE INDEX provides on apkindex (provides);" | sqlite3 $db + printf "CREATE INDEX deps on apkindex (deps);" | sqlite3 $db +} + +prepare() { + rm -f $db $csv + mkdir -p ${csv%/*} ${db%/*} } -printf "CREATE TABLE apkindex('name' text, 'version' text, 'desc' text, - 'url' text, 'lic' text, 'arch' text, 'deps' text, 'csum' text, - 'size' text, 'install_size' text, 'provides' text, 'install_if' text, - 'origin' text, 'maintainer' text, 'build_time' integer, 'commit' text);" | sqlite3 $database +rm -f $csv -IFS=$'\n' -while read line; do - [ "$line" = "" ] && write_line && reset_vars - read_line "$line" -done<$apkindex -unset IFS +for repo in $repos; do + for arch in $archs; do + rm -f /tmp/APKINDEX + #curl -s "$aports/$repo/$arch/APKINDEX.tar.gz" | tar -zx -C /tmp APKINDEX + tar -C /tmp -zxf $packages/$repo/$arch/APKINDEX.tar.gz APKINDEX + IFS=$'\n' + while read line; do + if [ "$line" = "" ]; then + write_line >> $csv + reset_vars + fi + read_line "$line" + done</tmp/APKINDEX + unset IFS + done +done -printf ".mode csv\n.import $csvfile apkindex" | sqlite3 $database +prepare && init_db && write_db |