commit 9a8c1d13b7fb063342c9b3e6e1ec524887389dc5
Author: magnum <magnum>
Date:   Thu Mar 10 14:41:27 2011 +0100

    FMT_UNICODE and FMT_UTF8 added. Now supports "--test --utf8" for testing all utf8-aware unicode formats. Also bails out if you try --utf8 on a non-aware format (that use Unicode)

diff --git a/src/NETLMv2_fmt.c b/src/NETLMv2_fmt.c
index 867815a..b67d110 100644
--- a/src/NETLMv2_fmt.c
+++ b/src/NETLMv2_fmt.c
@@ -298,7 +298,7 @@ struct fmt_main fmt_NETLMv2 = {
     SALT_SIZE,
     MIN_KEYS_PER_CRYPT,
     MAX_KEYS_PER_CRYPT,
-    FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE,
+    FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UNICODE | FMT_UTF8,
     tests
   }, {
     fmt_default_init,
diff --git a/src/NETNTLM_fmt.c b/src/NETNTLM_fmt.c
index 9f86319..7d5d1c6 100644
--- a/src/NETNTLM_fmt.c
+++ b/src/NETNTLM_fmt.c
@@ -191,7 +191,7 @@ struct fmt_main fmt_NETNTLM = {
     SALT_SIZE,
     MIN_KEYS_PER_CRYPT,
     MAX_KEYS_PER_CRYPT,
-    FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE,
+    FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UNICODE | FMT_UTF8,
     tests
   }, {
     fmt_default_init,
diff --git a/src/NETNTLMv2_fmt.c b/src/NETNTLMv2_fmt.c
index 18ee71c..c6dd737 100644
--- a/src/NETNTLMv2_fmt.c
+++ b/src/NETNTLMv2_fmt.c
@@ -347,7 +347,7 @@ struct fmt_main fmt_NETNTLMv2 = {
     SALT_SIZE_MAX,
     MIN_KEYS_PER_CRYPT,
     MAX_KEYS_PER_CRYPT,
-    FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE,
+    FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UNICODE | FMT_UTF8,
     tests
   }, {
     fmt_default_init,
diff --git a/src/NT_fmt.c b/src/NT_fmt.c
index 83b7b24..20feae6 100644
--- a/src/NT_fmt.c
+++ b/src/NT_fmt.c
@@ -845,7 +845,7 @@ struct fmt_main fmt_NT = {
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE,
+		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UNICODE | FMT_UTF8,
 		tests
 	}, {
 		fmt_NT_init,
diff --git a/src/bench.c b/src/bench.c
index 25409b7..dffba96 100644
--- a/src/bench.c
+++ b/src/bench.c
@@ -273,6 +273,17 @@ int benchmark_all(void)
 		if (!format->params.tests && format != fmt_list)
 			continue;
 
+/* Only test UTF-8 aware formats if --utf8 */
+		if ((options.flags & FLG_UTF8) && !(format->params.flags & FMT_UTF8)) {
+			if (options.format == NULL)
+				continue;
+			else
+				if (format->params.flags & FMT_UNICODE) {
+					printf("This format does not yet support UTF-8 conversion.\n\n");
+					continue;
+				}
+		}
+
 DoAgainWithoutNext:;
 		if (!strcmp(format->params.label, "md5-gen"))
 		{
diff --git a/src/formats.h b/src/formats.h
index 3edf034..9807850 100644
--- a/src/formats.h
+++ b/src/formats.h
@@ -19,6 +19,8 @@
  */
 #define FMT_CASE			0x00000001
 #define FMT_8_BIT			0x00000002
+#define FMT_UNICODE			0x00000004 // Uses Unicode (but not UTF-8) internally
+#define FMT_UTF8			0x00000008 // Supports the --utf8 flag
 #define FMT_BS				0x00010000
 #define FMT_SPLIT_UNIFIES_CASE		0x00020000
 
diff --git a/src/john.c b/src/john.c
index 5bf5faf..0ad109c 100644
--- a/src/john.c
+++ b/src/john.c
@@ -208,6 +208,8 @@ static void john_log_format(void)
 			"tried in chunks of %d",
 			min_chunk > 1 ? "will" : "may",
 			chunk);
+	if (options.flags & FLG_UTF8 && database.format->params.flags & FMT_UTF8)
+		log_event("- UTF-8 conversion enabled");
 }
 
 static char *john_loaded_counts(void)
@@ -303,6 +305,13 @@ static void john_load(void)
 			ldr_load_pw_file(&database, current->data);
 		} while ((current = current->next));
 
+		if (options.flags & FLG_UTF8 &&
+		    database.format->params.flags & FMT_UNICODE &&
+		    !(database.format->params.flags & FMT_UTF8)) {
+			fprintf(stderr, "This format does not yet support UTF-8 conversion\n");
+			error();
+		}
+
 		if ((options.flags & FLG_CRACKING_CHK) &&
 		    database.password_count) {
 			log_init(LOG_NAME, NULL, options.session);
diff --git a/src/mscash2_fmt.c b/src/mscash2_fmt.c
index 0f02651..74dfd5f 100644
--- a/src/mscash2_fmt.c
+++ b/src/mscash2_fmt.c
@@ -1432,7 +1432,7 @@ struct fmt_main fmt_mscash2 = {
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE,
+		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UNICODE | FMT_UTF8,
 		tests
 	}, {
 		init,
diff --git a/src/mscash_fmt.c b/src/mscash_fmt.c
index b1dadf6..fe04c3c 100644
--- a/src/mscash_fmt.c
+++ b/src/mscash_fmt.c
@@ -653,7 +653,7 @@ struct fmt_main fmt_mscash = {
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE,
+		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UNICODE | FMT_UTF8,
 		tests
 	}, {
 		init,
diff --git a/src/mssql05_fmt.c b/src/mssql05_fmt.c
index 089f134..277fecb 100644
--- a/src/mssql05_fmt.c
+++ b/src/mssql05_fmt.c
@@ -288,7 +288,7 @@ struct fmt_main fmt_mssql05 = {
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_8_BIT,
+		FMT_8_BIT | FMT_UNICODE,
 		mssql05_tests
 	}, {
 		mssql05_init,
diff --git a/src/mssql_fmt.c b/src/mssql_fmt.c
index 49abdd4..bb8eeb8 100644
--- a/src/mssql_fmt.c
+++ b/src/mssql_fmt.c
@@ -301,7 +301,7 @@ struct fmt_main fmt_mssql = {
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_8_BIT,
+		FMT_8_BIT | FMT_UNICODE,
 		mssql_tests
 	}, {
 		mssql_init,
diff --git a/src/rawMD5unicode_fmt.c b/src/rawMD5unicode_fmt.c
index 9b3a775..20868d6 100644
--- a/src/rawMD5unicode_fmt.c
+++ b/src/rawMD5unicode_fmt.c
@@ -380,7 +380,7 @@ struct fmt_main fmt_rawMD5unicode =
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_CASE | FMT_8_BIT,
+		FMT_CASE | FMT_8_BIT | FMT_UNICODE | FMT_UTF8,
 		rawmd5_tests
 	}, {
 		rawmd5_init,
diff --git a/src/sapG_fmt.c b/src/sapG_fmt.c
index 444f963..6886be4 100644
--- a/src/sapG_fmt.c
+++ b/src/sapG_fmt.c
@@ -31,6 +31,7 @@ unsigned char theMagicArray[MAGIC_ARRAY_SIZE]=
 #include "common.h"
 #include "formats.h"
 #include "sha.h"
+#include "options.h"
 
 #define FORMAT_LABEL			"sapg" 
 #define FORMAT_NAME			"SAP CODVN G (PASSCODE)"
@@ -191,6 +192,11 @@ void strToUpper_G(char* str)
  * translation for passwords (chars >7bit ascii).
  * translates just some >7bit chars to some weird magic bytes (not unicode?!)
  * works for most west-european (e.g. french) passwords...
+ *
+ * Addendum by magnum:
+ * This is actually UTF-8. The format uses UTF-8 internally so I add
+ * the FMT_UTF8 to it, and skip calling this ISO-8859-1 translation in case
+ * the --utf8 option is used
  */
 static char* translatePassword(char* origPassword)
 {
@@ -251,9 +257,14 @@ static void sapcodvng_crypt_all(int count) {
 	size_t secondHashDataLen;
 	size_t pwLen, unLen;
 	char tempVar[PLAINTEXT_LENGTH*2]; 	//PLAINTEXT_LENGTH is maxlen of password AND maxlen of username
-	
+	char* trPassword;
+
 	//0.	translate special characters in the password
-	char* trPassword=translatePassword(saved_key);
+	if (options.flags & FLG_UTF8)
+		trPassword = saved_key;
+	else
+		trPassword=translatePassword(saved_key);
+
 	pwLen=strlen(trPassword);
 
 	unLen=calcActualSaltSize_G(theSalt);
@@ -350,7 +361,7 @@ struct fmt_main fmt_sapG = {
 		SALT_SIZE,
 		MIN_KEYS_PER_CRYPT,
 		MAX_KEYS_PER_CRYPT,
-		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE, 
+		FMT_CASE | FMT_8_BIT | FMT_SPLIT_UNIFIES_CASE | FMT_UTF8,
 		sapcodvng_tests
 	}, {
 		sapcodvng_init,