Merge commit 'ce7b05ccae4549ca6e1ae2de1d3e36a8b3b82690' as 'modules/uwd/mysql-tools'

modules/uwd/mysql-tools/.gitignore

@@ -0,0 +1,13 @@
+/mysql_config
+mysql_tools_(for usr-local-bin)
+
+#
+# Editors / debuggers / other output files
+#
+*~
+*.bak
+*.orig
+*.patch
+callgrind.out.*
+
+.upt.json

modules/uwd/mysql-tools/README

@@ -0,0 +1,53 @@
+* Copyright (C) 2007 - 2015 Hyperweb2 All rights reserved.
+* GNU General Public License version 3; see www.hyperweb2.com/terms/
+
+#
+# HOW TO USE MYSQL_* tools
+#
+
+Before starting to dump/import, please rename mysql_config.dist in mysql_config and change infos using your configurations
+If you are using AppArmor , edit /etc/apparmor.d/usr.sbin.mysqld and add directory that must be used by the dump
+
+to easy dump/import the database, use ./mysql-tools command that includes dump and import scripts, instead if you need to separate the functions ,
+ use ./mysql-dump and ./mysql-import.
+
+mysql-tools utilization:
+
+#  This script automatically dump and import sql files
+#
+#	syntax: ./mysql_tools <opt> <database> <tables> <method> <configpath>
+#	OPT -> dump/import   ( export / import tables )
+#	TABLES -> use "," to separate table names; ex: mysql_dump "" "table1,table2..tableN". Leave blank to dump all tables.
+#	DATABASE -> specify database to process, use "" for default ( see in mysql-config )
+#	METHOD -> 0/1 (0: no fulldb,1: use fulldb) default: ( see in mysql-config )
+#	CONFIGPATH -> specify alternative path of config file, use "" to check in source folder
+#
+
+
+mysql-dump utilization: 
+
+# This script will export all tables from specified db and tables in separated sql files
+# it can also export the full db in a single sql file.
+#
+# syntax: ./mysql_dump <database> <tables> <method> <configpath>
+# options are not required, because values can be defined in config file:
+# DATABASE -> specify database to dump, use "" for default ( see in mysql-config )
+# TABLES -> use "" to dump all tables. Use "," to separate table names; ex: mysql_dump "" "table1,table2..tableN" 
+# METHOD -> 0/1 (0: no fulldb,1: with fulldb) default: ( see in mysql-config )
+# CONFIGPATH -> specify alternative path for config file, use "" to check in source folder.
+#
+
+
+mysql-import utilization:
+
+# This script will import the contents of the sql/ directory to the MySQL database.
+# You can choose to import table by table, or the entire db
+#
+# syntax: ./mysql_import <database> <tables> <method> <configpath>
+# options are not required, because values can be defined in config file:
+#
+# DATABASE -> specify database to dump, use "" for default ( see in mysql-config )
+# TABLES -> use "," to separate table names; ex: mysql_dump "" "table1,table2..tableN"    ..OR use "" to dump all tables.
+# METHOD -> 1/0  ( 1: by folder, 0: by full dump) default: ( see in mysql-config )
+# CONFIGPATH -> specify alternative path for config file, use "" to check in source folder.
+#

modules/uwd/mysql-tools/build-dump-parser.sh

@@ -0,0 +1,5 @@
+#!/bin/bash 
+
+gcc -O2 -Wall -pedantic dump-parser.c -o ./bin/dump-parser
+
+read -p "done"

modules/uwd/mysql-tools/dump-parser.c

@@ -0,0 +1,129 @@
+/* gcc -O2 -Wall -pedantic dump-parser.c -o dump-parser
+ Usage: cat dump.sql | dump-parser
+   Or : dump-parser dump.sql
+ bugs :
+ * the parser will fail if the 10001st character of a line is an escaped quote, it will see it as an unescaped quote.
+*/
+ 
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+ 
+#define BUFFER 100000
+ 
+bool is_escaped(char* string, int offset) {
+    if (offset == 0) {
+        return false;
+    } else if (string[offset - 1] == '\\') {
+        return !is_escaped(string, offset - 1);
+    } else {
+        return false;
+    }
+}
+ 
+bool is_commented(char* string) {
+    char buffer[4];
+ 
+    sprintf(buffer, "%.3s", string);
+ 
+    return strcmp(buffer, "-- ") == 0;
+}
+ 
+int main(int argc, char *argv[])
+{
+    FILE* file = argc > 1 ? fopen(argv[1], "r") : stdin;
+ 
+    char buffer[BUFFER];
+    char* line;
+    int pos;
+    int parenthesis = 0;
+    bool quote = false;
+    bool escape = false;
+    bool comment = false;
+ 
+    while (fgets(buffer, BUFFER, file) != NULL) {
+        line = buffer;
+ 
+        /* skip commented */
+        if (comment || is_commented(line)) {
+            comment = line[strlen(line) - 1] != '\n';
+            fputs(line, stdout);
+        } else {
+            pos = 0;
+ 
+            nullchar:
+            while (line[pos] != '\0') {
+                /* if we are still in escape state, we need to check first char. */
+                if (!escape) {
+                     /* find any character in ()' */
+                    pos = strcspn(line, "()'\\");
+                }
+ 
+                if (pos > 0) {
+                    /* print before match */
+                    printf("%.*s", pos, line);
+                }
+ 
+                switch (line[pos]) {
+                    case '(':
+                        if (!quote) {
+                            if (parenthesis == 0) {
+                                putchar('\n');
+                            }
+                            parenthesis++;
+                        }
+                        if (escape) {
+                            escape = false;
+                        }
+                        break;
+ 
+                    case ')':
+                        if (!quote) {
+                            if (parenthesis > 0) {
+                                parenthesis--;
+                            } else {
+                                /* whoops */
+                                puts("\n");
+                                fputs(line, stdout);
+                                fputs("Found closing parenthesis without opening one.\n", stderr);
+                                exit(1);
+                            }
+                        }
+                        if (escape) {
+                            escape = false;
+                        }
+                        break;
+ 
+                    case '\\':
+                        escape = !escape;
+                        break;
+ 
+                    case '\'':
+                        if (escape) {
+                            escape = false;
+                        } else {
+                            quote = !quote;
+                        }
+                        break;
+ 
+                    case '\0':
+                        goto nullchar;
+ 
+                    default:
+                        if (escape) {
+                            escape = false;
+                        }
+                        break;
+                }
+ 
+                /* print char then skip it (to make sure we don’t double match) */
+                putchar(line[pos]);
+                line = line + pos + 1;
+                pos = 0;
+            }
+        }
+    }
+ 
+    return 0;
+}

modules/uwd/mysql-tools/mysql-config.dist

@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# * Copyright (C) 2007 - 2015 Hyperweb2 All rights reserved.
+# * GNU General Public License version 3; see www.hyperweb2.com/terms/
+#
+# This file contains login/password information for accessing the MySQL database
+# and is used by all the mysql_* scripts.
+#
+
+#
+#	MYSQL
+#
+
+# change these lines with your mysql config
+MYSQL_DB=test
+MYSQL_USER=usr
+MYSQL_PASS=pwd
+MYSQL_HOST=localhost
+#MYSQL_SOCK=/var/lib/mysql/mysql.sock
+
+#
+#	File Options
+#
+
+# path of directory where extract separated tables ( without end slash )
+TPATH=./tables
+
+# (boolean) clean directory before dump, in this way non-existant db tables will be deleted
+CLEANFOLDER=1
+
+# path of file to extract database full dump
+FPATH=./full/full.sql
+
+# (boolean) switch to enable(1)/disable(0) the dump/import of full db file
+# ( you can do it manually using command parameters )
+FULL=0
+
+# (boolean) set 1 to enable --tab option for mysqldump and import data from it
+# it's very fast import/export process but doesn't utilize the insert query
+# NOTE: full db continue to be dumped with normal sql format
+# NOTE2: if you have problem with permissions ( mysql errorcode:13) mostly in linux
+# you should enable CHMODE config and disable/edit
+# some protections such as AppArmor in Ubuntu or SELinux in Fedora..
+
+TEXTDUMPS=1
+
+# (boolean) allow to change "TPATH" folder permissions to enable mysql server writing
+
+CHMODE=0
+
+
+#
+# TOOLS OPTIONS
+#
+
+#number of threads you want to use in TEXT import mode ( you can safely set it to your number of processor increasing process speed )
+THREADS=1
+
+IMPORTOPTS_TEXT="--use-threads=$THREADS --local --compress --delete --lock-tables"
+
+DUMPOPTS="--skip-comments --skip-set-charset --extended-insert --order-by-primary --single-transaction --quick"
+

modules/uwd/mysql-tools/mysql-dump

@@ -0,0 +1,119 @@
+#!/bin/bash
+#
+# * Copyright (C) 2007 - 2015 Hyperweb2 All rights reserved.
+# * GNU General Public License version 3; see www.hyperweb2.com/terms/
+
+echo "starting dump process.."
+# check config from same folder and include only if exists
+CONF_FILE=$MT_DIR"/mysql-config"
+if [ -f "$CONF_FILE" ]; then
+    source "$CONF_FILE"
+fi;
+
+#overwrite configs if file exists and variables are defined
+if [ ! -z "$4" ]; then
+    if [ -e "$4" ]; then
+        source "$4"
+    else # if 4th parameter is not a file, then try to eval
+        eval "$4"
+    fi;
+fi;
+
+source $MT_DIR"/shared-def"
+
+if [ ! -z "$1" ]; then 
+ MYSQL_DB=$1;
+fi
+
+#change group instead mod
+#group=`ls -l tables | awk '{print $4}'`
+#if [ $group != "mysql" ]; then
+#    if (($CHMODE != 0)); then
+#       sudo chgrp -v mysql $TPATH
+#    fi
+#fi
+
+#change permissions for other users
+if [ ! -z $TPATH ]; then
+	if [ ! -d "$TPATH" ]; then
+		#create the path recursively
+		echo "creating dir: $TPATH.."
+		mkdir -p "$TPATH"
+	fi
+	
+	if (($CHMODE != 0)); then
+		echo "changing permissions.."
+		sudo chmod -v o=rwx $TPATH
+	fi
+
+	#clean old tables
+	if (($CLEANFOLDER != 0)); then
+		rm -rvf $TPATH/*
+	fi
+	
+	if [ ! -z "$2" ]; then 
+	#if tables are specified in parameters then..
+		arr=$(echo $2 | tr "," "\n")
+
+		for T in $arr
+		do
+			echo "exporting "$T;
+			if (($TEXTDUMPS != 1)); then
+				FILE="$TPATH/$T.sql"
+				if [ -f $FILE ]; then
+					rm -f $FILE
+				fi
+
+                                if (($PARSEDUMP != 0)); then
+                                    echo "Parsing enabled";
+                                    eval "$MYSQLDUMP$OPTS $DUMPOPTS '$MYSQL_DB' '$T' | $DUMPPARSER > $FILE"
+                                else
+                                    eval "$MYSQLDUMP$OPTS $DUMPOPTS '$MYSQL_DB' '$T' -r $FILE"
+                                fi
+			else
+				echo $(eval "$MYSQLDUMP$OPTS $DUMPOPTS --tab=$TPATH/ '$MYSQL_DB' '$T'")
+			fi
+		done
+		
+	else 
+	#else get all tables from selected db
+			CMD="$MYSQL$OPTS -N -B -e 'show tables from \`$MYSQL_DB\`'"
+			echo "command: "$CMD
+			for T in $(eval $CMD)
+			do 
+				if (($TEXTDUMPS != 1)); then
+					echo "exporting "$T;
+					FILE="$TPATH/$T.sql"
+					if [ -f $FILE ]; then
+						rm -f $FILE
+					fi
+
+                                        if (($PARSEDUMP != 0)); then
+                                            echo "Parsing enabled on file "$FILE;
+                                            eval "$MYSQLDUMP$OPTS $DUMPOPTS '$MYSQL_DB' '$T' | $DUMPPARSER > $FILE"
+                                        else
+                                            eval "$MYSQLDUMP$OPTS $DUMPOPTS '$MYSQL_DB' '$T' -r $FILE"
+                                        fi
+				else
+					echo "exporting "$MYSQL_DB" tables: "$T;
+					echo $(eval "$MYSQLDUMP$OPTS $DUMPOPTS --tab=$TPATH/ '$MYSQL_DB' '$T'")
+				fi
+			done;
+
+	fi
+fi
+
+if [ ! -z "$3" ]; then FULL=$3; fi
+# export full file if option is enabled
+if (($FULL != 0)); then   
+	echo 'exporting FULL '$MYSQL_DB' in single file';
+	rm -f $FPATH
+
+        if (($PARSEDUMP != 0)); then
+            echo "Parsing enabled";
+            eval "$MYSQLDUMP$OPTS $DUMPOPTS '$MYSQL_DB' | $DUMPPARSER > $FPATH"
+        else
+            eval "$MYSQLDUMP$OPTS $DUMPOPTS '$MYSQL_DB' -r $FPATH"
+        fi
+fi
+

modules/uwd/mysql-tools/mysql-import

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# * Copyright (C) 2007 - 2015 Hyperweb2 All rights reserved.
+# * GNU General Public License version 3; see www.hyperweb2.com/terms/
+
+echo "starting import process.."
+# check config from same folder and include only if exists
+CONF_FILE=$MT_DIR"/mysql-config"
+if [ -f "$CONF_FILE" ]; then
+    source "$CONF_FILE"
+fi;
+
+#overwrite configs if file exists and variables are defined
+if [ ! -z "$4" ]; then
+    if [ -e "$4" ]; then
+        source "$4"
+    else # if 4th parameter is not a file, then try to eval
+        eval "$4"
+    fi;
+fi;
+
+source $MT_DIR"/shared-def"
+
+if [ ! -z "$1" ]; then 
+ MYSQL_DB=$1;
+fi
+
+# Table prefix ( to implement )
+PFX=
+
+# par 1 : db_name
+function checkdb {
+    # Check if database exists
+    err=`echo "quit" | $(eval "$MYSQL$OPTS '$1'") 2>&1`
+    if [ $? != 0 ]; then
+        if echo "$err" | grep -q "Access denied"; then
+            echo -e "\nDATABASE $1 EXISTS BUT USER $MYSQL_USER DOES NOT HAVE ACCESS TO IT, ABORTING"
+            exit
+        fi
+        echo -n "[creating $1]"
+        if ! echo "CREATE DATABASE $1;" | $(eval "$MYSQL$OPTS" )  2>/dev/null  ; then
+            echo -e "\nDATABASE $1 DOES NOT EXIST AND I FAILED TO CREATE IT, ABORTING"
+            exit
+        fi
+    fi
+}
+
+#par1: is_text_file ;  par2: file
+function import
+{
+    echo "Importing $2 into $MYSQL_DB (text: $1) ..."
+	
+	SQL="SET FOREIGN_KEY_CHECKS = 0;"
+	#eval "$MYSQL$OPTS -e \"SET FOREIGN_KEY_CHECKS = 0\" '$MYSQL_DB'" #disable foreign check
+	
+	FILE=$2	
+	if (($1 != 0)); then
+		#eval "$MYSQLIMPORT$OPTS $IMPORTOPTS_TEXT '$MYSQL_DB' $2"
+		TABLE=${FILE##*/}
+		TABLE=${TABLE%.txt}
+		SQL+="LOAD DATA LOCAL INFILE '$2' INTO TABLE $MYSQL_DB.$TABLE;"
+	else
+		#eval "$MYSQL$OPTS '$MYSQL_DB'" < "$2" 
+		SQL+="SOURCE $2;"
+	fi
+	#eval "$MYSQL$OPTS -e \"SET FOREIGN_KEY_CHECKS = 1\" '$MYSQL_DB'" #enable foreign check
+	SQL+="SET FOREIGN_KEY_CHECKS = 1";
+	
+	eval "$MYSQL$OPTS -e \"$SQL\" '$MYSQL_DB'"
+	
+	echo " done"
+}
+
+if [ ! -z "$1" ]; then 
+	DB=$1; 
+fi
+
+checkdb $MYSQL_DB
+
+if [ ! -z "$2" ]; then 
+	tables=$(echo $2 | tr "," "\n")
+fi
+if [ ! -z "$3" ]; then 
+	FULL=$3; 
+fi
+
+if (($FULL != 0)); then
+	echo "importing full world file"
+	$(eval "$MYSQL$OPTS '$MYSQL_DB'") < $FPATH
+else
+	if [ ! -z "$2" ]; then
+		import "0" "$TPATH/$2.sql" # TODO we should check if it's a set of tables before
+		if (($TEXTDUMPS != 0)); then
+			for x in $TPATH/*.txt; do
+				import "1" "$x"
+			done
+		fi
+	else
+		for x in $TPATH/*.sql; do
+			import "0" "$x"
+		done
+
+		if (($TEXTDUMPS != 0)); then
+			for x in $TPATH/*.txt; do
+				import "1" "$x"
+			done
+		fi
+	fi
+fi

modules/uwd/mysql-tools/mysql-tools

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# * Copyright (C) 2007 - 2015 Hyperweb2 All rights reserved.
+# * GNU General Public License version 3; see www.hyperweb2.com/terms/
+
+MT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+	if [ ! -z "$1" ]; then 
+		if [ "$1" == "dump" ]; then 
+			source $MT_DIR"/mysql-dump" "$3" "$2" "$4" "$5"
+			return
+		elif [ "$1" == "import" ]; then
+			source $MT_DIR"/mysql-import" "$3" "$2" "$4" "$5"
+			return
+		fi
+	else
+		echo "!!no command specified!!"
+	fi
+}
+
+# OPT - Tables - database - full file - config path
+main "$1" "$2" "$3" "$4" "$5"

modules/uwd/mysql-tools/shared-def

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# * Copyright (C) 2007 - 2015 Hyperweb2 All rights reserved.
+# * GNU General Public License version 3; see www.hyperweb2.com/terms/
+
+MYSQL="mysql"
+MYSQLIMPORT="mysqlimport"
+MYSQLDUMP="mysqldump"
+DUMPPARSER=$MT_DIR"/bin/dump-parser"
+
+#empty for bash commands
+WIN_BIN=0
+command -v mysql >/dev/null 2>&1 || { WIN_BIN=1; }
+command -v mysqlimport >/dev/null 2>&1 || { WIN_BIN=1; }
+command -v mysqldump >/dev/null 2>&1 || { WIN_BIN=1; }
+
+WIN_BIN=1
+if [ `uname -s` == "Linux" ]; then
+   WIN_BIN=0
+else
+   WIN_BIN=1 
+fi;
+
+if (($WIN_BIN != 0)); then
+	MYSQL=$MT_DIR"/bin/$MYSQL"
+	MYSQLIMPORT=$MT_DIR"/bin/$MYSQLIMPORT"
+	MYSQLDUMP=$MT_DIR"/bin/$MYSQLDUMP"
+fi;
+
+OPTS=
+[ ! -z "$MYSQL_USER" ] && OPTS+=" -u $MYSQL_USER"
+[ ! -z "$MYSQL_PASS" ] && OPTS+=" -p$MYSQL_PASS"
+[ ! -z "$MYSQL_HOST" ] && OPTS+=" -h $MYSQL_HOST"
+[ ! -z "$MYSQL_SOCK" ] && OPTS+=" -S $MYSQL_SOCK"

modules/uwd/mysql-tools/upt.json

@@ -0,0 +1,11 @@
+{
+    "name": "udw/mysql-tools",
+    "dependencies": {
+    },
+    "devDependencies": {
+    },
+    "_comment_keep": "ensure we keep the right git repository configurations",
+    "keep": [
+        ".git/config"
+    ]
+}