Enforced access controls on some administrative actions.

 - ensure use of HTTP POST method : HTTP GET should only be used for
information retrieval and not to perform server side effect operations
(see HTTP standard https://tools.ietf.org/html/rfc7231#section-4.2.1)
 - a transaction token is now required for these administrative form
submissions to ensure the request can not be included in an external
site and performed silently/by mistake by the user browser

6 files changed, 82 insertions, 5 deletions

diff --git a/bin/clearall.sh b/bin/clearall.sh
index 17cb50e99..885379c9c 100755
--- a/bin/clearall.sh
+++ b/bin/clearall.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-./apicall.sh "/IndexControlURLs_p.html?deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=on" > /dev/null
\ No newline at end of file
+./protectedPostApiCall.sh "IndexControlURLs_p.html" "deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=on"
\ No newline at end of file
diff --git a/bin/clearcache.sh b/bin/clearcache.sh
index 906595bec..926d70e78 100755
--- a/bin/clearcache.sh
+++ b/bin/clearcache.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-./apicall.sh "/IndexControlURLs_p.html?deleteIndex=off&deleteSolr=off&deleteCache=on&deleteCrawlQueues=off&deleteRobots=on&deleteSearchFl=on&deletecomplete=" > /dev/null
+./protectedPostApiCall.sh "IndexControlURLs_p.html" "deleteIndex=off&deleteSolr=off&deleteCache=on&deleteCrawlQueues=off&deleteRobots=on&deleteSearchFl=on&deletecomplete="
diff --git a/bin/clearindex.sh b/bin/clearindex.sh
index c32abf94a..9c78bb997 100755
--- a/bin/clearindex.sh
+++ b/bin/clearindex.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-./apicall.sh "/IndexControlURLs_p.html?deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=off" > /dev/null
+./protectedPostApiCall.sh "IndexControlURLs_p.html" "deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=off"
diff --git a/bin/deleteurl.sh b/bin/deleteurl.sh
index fd9949637..94aefcfb3 100755
--- a/bin/deleteurl.sh
+++ b/bin/deleteurl.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-./apicall.sh "/IndexControlURLs_p.html?urlhashdeleteall=&urlstring=$1" > /dev/null
+./protectedPostApiCall.sh "IndexControlURLs_p.html" "urlhashdeleteall=&urlstring=$1"
diff --git a/bin/passwd.sh b/bin/passwd.sh
index 62267fc46..45335853e 100755
--- a/bin/passwd.sh
+++ b/bin/passwd.sh
@@ -6,7 +6,7 @@ if [ -z "$1" ]; then
 	exit 2
 fi
 
-(./apicall.sh "ConfigAccounts_p.html?setAdmin=&adminuser=admin&adminpw1=$1&adminpw2=$1&access=" > /dev/null && \
+(./protectedPostApiCall.sh "ConfigAccounts_p.html" "setAdmin=&adminuser=admin&adminpw1=$1&adminpw2=$1&access=" && \
 echo "Password for User Name 'admin' set to '$1'") || \
 (echo "Password setting failed" && \
 exit 1)
\ No newline at end of file
diff --git a/bin/protectedPostApiCall.sh b/bin/protectedPostApiCall.sh
new file mode 100755
index 000000000..652abdf35
--- /dev/null
+++ b/bin/protectedPostApiCall.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env sh
+# Call a YaCy HTTP POST API URL protected by HTTP authentication and transaction token validation
+# $1 : API path
+# $2 : POST parameters (example : "param1=value1&param2=value2")
+#
+# Authentication options :
+# - enable unauthenticated local access as administrator : set adminAccountForLocalhost=true in the DATA/SETTINGS/yacy.conf file
+# - OR use the legacy Basic HTTP authentication mode (unsecured for remote access): set the "auth-method" to BASIC in the defaults/web.xml file
+# - OR use the Digest HTTP authentication mode : set the "auth-method" to DIGEST in the defaults/web.xml file. 
+#   With that last option, the script will run in interactive mode as default, prompting for the administrator password.
+#   To run in batch mode, you must first export an environment variable filled with the clear-text administrator password before using this script :
+#    For example with > export YACY_ADMIN_PASSWORD=your_admin_password
+# 
+
+cd "`dirname $0`"
+port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+admin=$(grep ^adminAccountUserName= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= ../DATA/SETTINGS/yacy.conf | cut -d= -f2)
+
+if grep "<auth-method>BASIC</auth-method>" ../defaults/web.xml > /dev/null; then
+	# When authentication method is in basic mode, use directly the password hash from the configuration file 
+	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+fi
+
+if which curl > /dev/null; then
+	if [ "$adminAccountForLocalhost" = "true" ]; then
+		# localhost access as administrator without authentication is enabled
+		
+		# retrieve the transaction token from the HTTP GET flavor of the URL 
+		transactionToken=$(curl -sSfI "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]')
+		# send POST data including the transaction token
+		curl -sSf -d "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1" > /dev/null
+		
+	else
+		if [ -z "$YACY_ADMIN_PASSWORD" ]; then
+			# no password environment variable : we ask interactively for it only once (not using read -s to be POSIX compliant)
+			stty -echo
+			read -p "Enter host password for user '$admin':" YACY_ADMIN_PASSWORD
+			stty echo
+			printf "\n"
+		fi
+		
+		# retrieve the transaction token from the HTTP GET flavor of the URL 
+		transactionToken=$(curl -sSfI --anyauth -u "$admin:$YACY_ADMIN_PASSWORD" "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]')
+		# send POST data including the transaction token
+		curl -sSf --anyauth -u "$admin:$YACY_ADMIN_PASSWORD" -d "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1" > /dev/null
+		
+	fi
+elif which wget > /dev/null; then
+	
+	if [ "$adminAccountForLocalhost" = "true" ]; then
+		# localhost access as administrator without authentication is enabled
+		
+		# retrieve the transaction token from the HTTP GET flavor of the URL
+		transactionToken=$(wget -q -t 1 -O - --save-headers --timeout=120 "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]')
+		# send POST data including the transaction token
+		wget -nv -t 1 -O /dev/null --timeout=120 --post-data "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1"
+		
+	else
+		if [ -z "$YACY_ADMIN_PASSWORD" ]; then
+			# no password environment variable : we ask interactively for it only once (not using read -s to be POSIX compliant)
+			stty -echo
+			read -p "Enter host password for user '$admin':" YACY_ADMIN_PASSWORD
+			stty echo
+			printf "\n"
+		fi
+
+		# retrieve the transaction token from the HTTP GET flavor of the URL
+		transactionToken=$(wget -q -t 1 -O - --http-user "$admin" --http-password "$YACY_ADMIN_PASSWORD" --save-headers --timeout=120 "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]')
+		# send POST data including the transaction token
+		wget -nv -t 1 -O /dev/null --timeout=120 --http-user "$admin" --http-password "$YACY_ADMIN_PASSWORD" --post-data "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1"
+		
+	fi
+else
+	printf "Please install curl or wget\n" > /dev/stderr
+	exit 1
+fi
\ No newline at end of file