main/abuild: fix fetch to nfs directory

3 files changed, 316 insertions, 52 deletions

diff --git a/main/abuild/0001-abuild-fix-fetch-lock-file-on-nfs.patch b/main/abuild/0001-abuild-fix-fetch-lock-file-on-nfs.patch
new file mode 100644
index 0000000000..b749f30769
--- /dev/null
+++ b/main/abuild/0001-abuild-fix-fetch-lock-file-on-nfs.patch
@@ -0,0 +1,311 @@
+From 7479ed1efcd4ac1f5cc93aa65352f6a25bcbf218 Mon Sep 17 00:00:00 2001
+From: Natanael Copa <ncopa@alpinelinux.org>
+Date: Wed, 26 Aug 2015 08:14:36 +0200
+Subject: [PATCH] abuild: fix fetch lock file on nfs
+
+flock(2) on an NFS mount will on the server side convert the lock to a
+POSIX lock (fcntl(F_SETLK)). This means that abuild running on NFS
+server and client will create different locks and they will both try
+download same file at same time.
+
+We fix this by creating a small abuild-fetch application that will
+create a POSIX lock which works with NFS.
+---
+ Makefile       |   6 +-
+ abuild-fetch.c | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ abuild.in      |  47 +--------------
+ 3 files changed, 190 insertions(+), 47 deletions(-)
+ create mode 100644 abuild-fetch.c
+
+diff --git a/Makefile b/Makefile
+index 81c2851..b7332b2 100644
+--- a/Makefile
++++ b/Makefile
+@@ -11,7 +11,7 @@ abuildrepo	?= ~/.cache/abuild
+ SCRIPTS		:= abuild abuild-keygen abuild-sign newapkbuild \
+ 		   abump apkgrel buildlab apkbuild-cpan checkapk \
+ 		   apkbuild-gem-resolver
+-USR_BIN_FILES	:= $(SCRIPTS) abuild-tar abuild-sudo
++USR_BIN_FILES	:= $(SCRIPTS) abuild-tar abuild-sudo abuild-fetch
+ SAMPLES		:= sample.APKBUILD sample.initd sample.confd \
+ 		sample.pre-install sample.post-install
+ AUTOTOOLS_TOOLCHAIN_FILES := config.sub
+@@ -47,6 +47,7 @@ LIBS-abuild-tar = $(SSL_LIBS)
+ CFLAGS-abuild-tar = $(SSL_CFLAGS)
+ 
+ OBJS-abuild-sudo = abuild-sudo.o
++OBJS-abuild-fetch = abuild-fetch.o
+ 
+ .SUFFIXES:	.sh.in .in
+ %.sh: %.sh.in
+@@ -73,6 +74,9 @@ abuild-sudo: abuild-sudo.o
+ abuild-tar: abuild-tar.o
+ 	$(LINK)
+ 
++abuild-fetch: abuild-fetch.o
++	$(LINK)
++
+ abuild-tar.static: abuild-tar.o
+ 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CFLAGS-$@) -o $@ -static $(LIBS-$@) $^
+ 
+diff --git a/abuild-fetch.c b/abuild-fetch.c
+new file mode 100644
+index 0000000..89d914f
+--- /dev/null
++++ b/abuild-fetch.c
+@@ -0,0 +1,184 @@
++/* MIT license
++
++Copyright (C) 2015 Natanael Copa <ncopa@alpinelinux.org>
++
++Permission is hereby granted, free of charge, to any person obtaining a copy
++of this software and associated documentation files (the "Software"), to deal
++in the Software without restriction, including without limitation the rights
++to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
++copies of the Software, and to permit persons to whom the Software is
++furnished to do so, subject to the following conditions:
++
++The above copyright notice and this permission notice shall be included in
++all copies or substantial portions of the Software.
++
++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
++AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
++THE SOFTWARE.
++
++*/
++
++
++#include <sys/wait.h>
++
++#include <err.h>
++#include <fcntl.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <string.h>
++#include <unistd.h>
++
++static char *program;
++static char lockfile[PATH_MAX] = "";
++
++struct cmdarray {
++	size_t argc;
++	char *argv[32];
++};
++
++void add_opt(struct cmdarray *cmd, char *opt)
++{
++	cmd->argv[cmd->argc++] = opt;
++	cmd->argv[cmd->argc] = NULL;
++}
++
++int usage(int eval)
++{
++	printf("usage: %s [-d DESTDIR] URL\n", program);
++	return eval;
++}
++
++/* create or wait for an NFS-safe lockfile and fetch url with curl or wget */
++int fetch(char *url, const char *destdir)
++{
++	int lockfd, status=0;
++	pid_t childpid;
++	char outfile[PATH_MAX], partfile[PATH_MAX];
++	char *name;
++	struct flock fl = {
++		.l_type = F_WRLCK,
++		.l_whence = SEEK_SET,
++		.l_start = 1,
++		.l_len = 0,
++	};
++	struct cmdarray curlcmd = {
++		.argc = 6,
++		.argv = { "curl", "-k", "-L", "-f", "-o", partfile, NULL }
++	};
++	struct cmdarray wgetcmd = {
++		.argc = 3,
++		.argv = { "wget", "-O", partfile, NULL }
++	};
++	name = strrchr(url, '/');
++	if (name == NULL)
++		errx(1, "%s: no '/' in url");
++
++	snprintf(outfile, sizeof(outfile), "%s%s", destdir, name);
++	snprintf(lockfile, sizeof(lockfile), "%s.lock", outfile);
++	snprintf(partfile, sizeof(partfile), "%s.part", outfile);
++
++	lockfd = open(lockfile, O_WRONLY|O_CREAT, 0660);
++	if (lockfd < 0)
++		err(1, lockfile);
++
++	if (fcntl(lockfd, F_SETLK, &fl) < 0) {
++		printf("Waiting for %s ...\n", lockfile);
++		if (fcntl(lockfd, F_SETLKW, &fl) < 0)
++			err(1, "fcntl(F_SETLKW)");
++	}
++
++	if (access(outfile, F_OK) == 0) {
++		printf("%s already exist. skipping\n", outfile);
++		goto fetch_done;
++	}
++
++	if (access(partfile, F_OK) == 0) {
++		printf("Partial download found. Trying to resume.\n");
++		add_opt(&curlcmd, "-C");
++		add_opt(&curlcmd, "-");
++		add_opt(&wgetcmd, "-c");
++	}
++
++	add_opt(&curlcmd, url);
++	add_opt(&wgetcmd, url);
++
++	childpid = fork();
++	if (childpid < 0 )
++		err(1, "fork");
++
++	if (childpid == 0) {
++		execvp(curlcmd.argv[0], curlcmd.argv);
++		printf("Using wget\n");
++		execvp(wgetcmd.argv[0], wgetcmd.argv);
++		warn(wgetcmd.argv[0]);
++		unlink(lockfile);
++		exit(1);
++	}
++	wait(&status);
++	rename(partfile, outfile);
++
++fetch_done:
++	unlink(lockfile);
++	close(lockfd);
++	lockfile[0] = '\0';
++	return status;
++
++}
++
++void sighandler(int sig)
++{
++	switch(sig) {
++	case SIGABRT:
++	case SIGINT:
++	case SIGQUIT:
++	case SIGTERM:
++		unlink(lockfile);
++		exit(0);
++		break;
++	default:
++		break;
++	}
++}
++
++int main(int argc, char *argv[])
++{
++	int opt, r=0, i;
++	char *destdir = "/var/cache/distfiles";
++
++	program = argv[0];
++	while ((opt = getopt(argc, argv, "hd:")) != -1) {
++		switch (opt) {
++		case 'h':
++			return usage(0);
++			break;
++		case 'd':
++			destdir = optarg;
++			break;
++		default:
++			printf("Unkonwn option '%c'\n", opt);
++			return usage(1);
++			break;
++		}
++	}
++
++	argv += optind;
++	argc -= optind;
++
++	if (argc < 1)
++		return usage(1);
++
++	signal(SIGABRT, sighandler);
++	signal(SIGINT, sighandler);
++	signal(SIGQUIT, sighandler);
++	signal(SIGTERM, sighandler);
++
++	for (i = 0; i < argc; i++) {
++		if (fetch(argv[i], destdir))
++			r++;
++	}
++	return r;
++}
+diff --git a/abuild.in b/abuild.in
+index b8d8851..6e96187 100644
+--- a/abuild.in
++++ b/abuild.in
+@@ -298,23 +298,6 @@ sourcecheck() {
+ 	return 0
+ }
+ 
+-# convert curl options to wget options and call wget instead of curl
+-wget_fallback() {
+-	local wget_opts= outfile= opt=
+-	while getopts "C:Lko:s" opt; do
+-		case $opt in
+-		'L') ;; # --location. wget does this by default
+-		'f') ;; # --fail. wget does this by default
+-		'C') wget_opts="$wget_opts -c";; # --continue-at
+-		's') wget_opts="$wget_opts -q";; # --silent
+-		'o') wget_opts="$wget_opts -O $OPTARG";; # --output
+-		'k') wget_opts="$wget_opts --no-check-certificate";; #gnu wget
+-		esac
+-	done
+-	shift $(( $OPTIND - 1 ))
+-	wget $wget_opts "$1"
+-}
+-
+ uri_fetch() {
+ 	local uri="$1"
+ 	local d="${uri##*/}"	# $(basename $uri)
+@@ -335,38 +318,10 @@ uri_fetch() {
+ 			;;
+ 	esac
+ 
+-	case "$uri" in
+-		https://*) opts="-k";;
+-	esac
+-
+ 	mkdir -p "$SRCDEST"
+ 
+ 	CLEANUP_FILES="$CLEANUP_FILES $lockfile"
+-	(
+-	flock -n -x 9 || msg "Waiting for ${lockfile##*/}..."
+-	flock -x 9
+-
+-	[ -f "$SRCDEST/$d" ] && exit 0 # use exit since its a subshell
+-
+-	if [ -f "$SRCDEST/$d.part" ]; then
+-		msg "Partial download found. Trying to resume"
+-		opts="$opts -C -"
+-	fi
+-	msg "Fetching $uri"
+-
+-	# fallback to wget if curl is missing. useful for bootstrapping
+-	local fetcher=
+-	if ! [ -x "$(which curl)" ]; then
+-		fetcher=wget_fallback
+-	else
+-		fetcher=curl
+-		opts="$opts -L -f -k"
+-	fi
+-
+-	$fetcher $opts -o "$SRCDEST/$d.part" "$uri" \
+-		&& mv "$SRCDEST/$d.part" "$SRCDEST/$d"
+-
+-	) 9>$lockfile
++	abuild-fetch -d "$SRCDEST" "$uri"
+ 
+ 	local rc=$?
+ 	rm -f "$lockfile"
+-- 
+2.5.0
+
diff --git a/main/abuild/0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch b/main/abuild/0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch
deleted file mode 100644
index 83a79f09b6..0000000000
--- a/main/abuild/0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch
+++ /dev/null
@@ -1,47 +0,0 @@
-From 8468825035d1c0af12abbb67aea85260cd048946 Mon Sep 17 00:00:00 2001
-From: Natanael Copa <ncopa@alpinelinux.org>
-Date: Wed, 26 Aug 2015 08:14:36 +0200
-Subject: [PATCH] abuild: workaround with fetch lock file on nfs
-
-flock(2) on an NFS mount will on the server side convert the lock to a
-POSIX lock (fcntl(F_SETLK)). This means that abuild running on NFS
-server and client will create different locks and they will both try
-download same file at same time.
-
-The proper fix would be to patch curl to have an option to create a
-posix lock for the downlaoded file or create a flock -c like wrapper
-that creates posix lock.
-
-Meanwhile, we implement a stupid test for existance of the lockfile
-before we go on. This is not race-safe because other process could
-create the lockfile after the [ -e $lockfile ] test and before the file
-is created.
-
-At least it is not worse that no locking at all.
----
- abuild.in | 9 +++++++++
- 1 file changed, 9 insertions(+)
-
-diff --git a/abuild.in b/abuild.in
-index b8d8851..f01fe2b 100644
---- a/abuild.in
-+++ b/abuild.in
-@@ -342,6 +342,15 @@ uri_fetch() {
- 	mkdir -p "$SRCDEST"
- 
- 	CLEANUP_FILES="$CLEANUP_FILES $lockfile"
-+
-+	# best-effort workaround for NFS which has issues with flock
-+	if [ -e "$lockfile" ]; then
-+		msg "Waiting for ${lockfile##*/}..."
-+		while [ -e "$lockfile" ]; do
-+			sleep 0.5s
-+		done
-+	fi
-+
- 	(
- 	flock -n -x 9 || msg "Waiting for ${lockfile##*/}..."
- 	flock -x 9
--- 
-2.5.0
-
diff --git a/main/abuild/APKBUILD b/main/abuild/APKBUILD
index b26f8f6c32..10fd0e1eb3 100644
--- a/main/abuild/APKBUILD
+++ b/main/abuild/APKBUILD
@@ -2,7 +2,7 @@
 pkgname=abuild
 pkgver=2.24.0
 _ver=${pkgver%_git*}
-pkgrel=1
+pkgrel=2
 pkgdesc="Script to build Alpine Packages"
 url="http://git.alpinelinux.org/cgit/abuild/"
 arch="all"
@@ -20,7 +20,7 @@ subpackages="apkbuild-cpan:cpan apkbuild-gem-resolver:gems"
 options="suid"
 pkggroups="abuild"
 source="http://dev.alpinelinux.org/archive/abuild/abuild-$_ver.tar.xz
-	0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch
+	0001-abuild-fix-fetch-lock-file-on-nfs.patch
 	"
 
 _builddir="$srcdir/$pkgname-$_ver"
@@ -66,8 +66,8 @@ gems() {
 }
 
 md5sums="7821eea872e86a768d881fecce95e934  abuild-2.24.0.tar.xz
-f8ae835a70e5accbc1a02d3daf470a23  0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch"
+1de97f36c186c85c6381e731af9359f9  0001-abuild-fix-fetch-lock-file-on-nfs.patch"
 sha256sums="0e0c0c7579c3b8d6668fad9d93a42f3ea18da88679e1a12a178f2676c79e94d6  abuild-2.24.0.tar.xz
-4438e69a4e23beec50e3f6def7c91b1ca7c12b1ff71c3e918fc4b4cab1976136  0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch"
+9a2dfb3adffd5ae7b2b824ed1e099f0c69eb1d9e34550245f4465d87210b414c  0001-abuild-fix-fetch-lock-file-on-nfs.patch"
 sha512sums="360d0164afed9c61576618d113d738327151c60f1bc1e6246fcf74f97db087ca2effb11a4ae045f1f288063d5311c6dddbe7721b3cbe65d11472b3d2b0f03ef6  abuild-2.24.0.tar.xz
-1b0481f70cc3a28990985eded62a62c64b261a785278b765cc275f844b32859acce59478170f08242de2f2c89f97e848f792271102be3d6dea0c94d6deea03c0  0001-abuild-workaround-with-fetch-lock-file-on-nfs.patch"
+c371afe46fba92586550f7554972f58b3e7f734e42dc138d7cb8fa781b50ced2a3b7cf1795b51ed2846ea6702377f3a436d055a12908faf70eb92f253f3ce3cb  0001-abuild-fix-fetch-lock-file-on-nfs.patch"