Openwall GNU/*/Linux - a small security-enhanced Linux distro for servers
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Date: Fri, 22 Jan 2016 18:24:08 +0100
From: magnum <john.magnum@...hmail.com>
To: john-dev@...ts.openwall.com
Subject: Re: loader and cracker (prefetch) optimizations from
 September 2015

On 2016-01-21 12:08, Solar Designer wrote:
> FYI, I've just committed the loader and cracker optimizations from
> September 2015 to the core tree.  There are a few differences from what
> went into jumbo back then, but almost all of those are on purpose, and
> should remain as differences.  For example, show_uid_in_cracks is
> jumbo-specific, which affected these changes in a few places.  So when
> merging these, you should mostly keep the code currently in jumbo as-is,
> even if the new core code does those things differently.
>
> One exception to that is SSE2 vs. SSE checks for the prefetching.
> It turns out those prefetch instructions and intrinsics are available
> with plain SSE (Pentium 3) rather than require SSE2 (Pentium 4), so
> let's in fact be checking __SSE__ and #include'ing <xmmintrin.h>, rather
> than checking __SSE2__ and #include'ing <emmintrin.h>.

Thanks for pointing out how to merge this, it really made it easy. I 
think I got it right: Attached is the net changes to Jumbo.

I did not "reduce CRACKED_HASH_LOG from jumbo's 25 to 21"... should we 
do so in Jumbo?

magnum

diff --git a/doc/CHANGES b/doc/CHANGES
index 8549ae6..cb0ae8e 100644
--- a/doc/CHANGES
+++ b/doc/CHANGES
@@ -1,12 +1,17 @@
-The following changes have been made between John 1.8.0 and 1.8.0.6:
+The following changes have been made between John 1.8.0 and 1.8.0.7:
 
 * 3x interleaving for bcrypt on x86-64, except in AVX-enabled builds.
 Provides significant speedup on Core 2 CPUs.
 * Recognize the $2b$ bcrypt prefix.
 * In the generic crypt(3) format, detect descrypt with valid vs. invalid salts
 as separate id's for our heuristics on supported hash types.
-* When built with --fork support, disallow session names with all-digit
-suffixes since these clash with those produced by --fork.
+* Introduced a number of optimizations for faster handling of large password
+hash files, including loading, cracking, and "--show".  Some of these use more
+memory than before, yet in a more efficient manner.
+* Implemented special-case handling of repeated rule commands '$', '^', '[',
+']', '{', and '}', as well as faster handling of the 'D' command.
+* When built with "--fork" support, disallow session names with all-digit
+suffixes since these clash with those produced by "--fork".
 * Forward SIGTERM to --fork'ed children.
 * Set stdout to line buffered (rather than potentially fully buffered), except
 for "--stdout", "--show", and auxiliary programs such as "unshadow".
@@ -23,6 +28,7 @@ of NEON for bitslice DES).
 be generated by GCC 4.6+ in the bitslice DES code in non-OpenMP builds.
 * Fixed a bug where "Warning: no OpenMP support for this hash type" could be
 printed in "--stdout" mode.
+* Made assorted other bugfixes, portability and documentation enhancements.
 
 The following changes have been made between John 1.7.9.8 and 1.8.0:
 
diff --git a/src/cracker.c b/src/cracker.c
index b539a06..ed668b4 100644
--- a/src/cracker.c
+++ b/src/cracker.c
@@ -29,9 +29,14 @@
 #endif
 
 #include "arch.h"
+#include "params.h"
+
+#if CRK_PREFETCH && defined(__SSE__)
+#include <xmmintrin.h>
+#endif
+
 #include "misc.h"
 #include "math.h"
-#include "params.h"
 #include "memory.h"
 #include "signals.h"
 #include "idle.h"
@@ -775,7 +780,7 @@ static int crk_password_loop(struct db_salt *salt)
 			unsigned int *b = &salt->bitmap[h / (sizeof(*salt->bitmap) * 8)];
 			a[slot].i = h;
 			a[slot].u.b = b;
-#ifdef __SSE2__
+#ifdef __SSE__
 			_mm_prefetch((const char *)b, _MM_HINT_NTA);
 #else
 			*(volatile unsigned int *)b;
@@ -786,7 +791,7 @@ static int crk_password_loop(struct db_salt *salt)
 			unsigned int h = a[slot].i;
 			if (*a[slot].u.b & (1U << (h % (sizeof(*salt->bitmap) * 8)))) {
 				struct db_password **pwp = &salt->hash[h >> PASSWORD_HASH_SHR];
-#ifdef __SSE2__
+#ifdef __SSE__
 				_mm_prefetch((const char *)pwp, _MM_HINT_NTA);
 #else
 				*(void * volatile *)pwp;
@@ -803,9 +808,9 @@ static int crk_password_loop(struct db_salt *salt)
 /*
  * Chances are this will also prefetch the next_hash field and the actual
  * binary (pointed to by the binary field, but likely located right after
- * this struct.
+ * this struct).
  */
-#ifdef __SSE2__
+#ifdef __SSE__
 			_mm_prefetch((const char *)&pw->binary, _MM_HINT_NTA);
 #else
 			*(void * volatile *)&pw->binary;
diff --git a/src/loader.h b/src/loader.h
index 39dc3b0..937bad1 100644
--- a/src/loader.h
+++ b/src/loader.h
@@ -1,6 +1,6 @@
 /*
  * This file is part of John the Ripper password cracker,
- * Copyright (c) 1996-98,2010-2013 by Solar Designer
+ * Copyright (c) 1996-98,2010-2013,2015 by Solar Designer
  *
  * ...with changes in the jumbo patch, by various authors
  */
diff --git a/src/memory.c b/src/memory.c
index 03952c9..de27379 100644
--- a/src/memory.c
+++ b/src/memory.c
@@ -1,6 +1,6 @@
 /*
  * This file is part of John the Ripper password cracker,
- * Copyright (c) 1996-98,2010,2012 by Solar Designer
+ * Copyright (c) 1996-98,2010,2012,2016 by Solar Designer
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted.
@@ -83,7 +83,8 @@ void *mem_alloc_func(size_t size
 {
 	void *res;
 
-	if (!size) return NULL;
+	if (!size)
+		return NULL;
 #if defined (MEMDBG_ON)
 	res = (char*) MEMDBG_alloc(size, file, line);
 #else
@@ -98,7 +99,7 @@ void *mem_alloc_func(size_t size
 	return res;
 }
 
-void *mem_calloc_func(size_t count, size_t size
+void *mem_calloc_func(size_t nmemb, size_t size
 #if defined (MEMDBG_ON)
 	, char *file, int line
 #endif
@@ -106,16 +107,17 @@ void *mem_calloc_func(size_t count, size_t size
 {
 	void *res;
 
-	if (!count || !size) return NULL;
+	if (!nmemb || !size)
+		return NULL;
 #if defined (MEMDBG_ON)
-	size *= count;
+	size *= nmemb;
 	res = (char*) MEMDBG_alloc(size, file, line);
 	memset(res, 0, size);
 #else
-	res = calloc(count, size);
+	res = calloc(nmemb, size);
 #endif
 	if (!res) {
-		fprintf(stderr, "mem_calloc(): %s trying to allocate "Zu" bytes\n", strerror(ENOMEM), count * size);
+		fprintf(stderr, "mem_calloc(): %s trying to allocate "Zu" bytes\n", strerror(ENOMEM), nmemb * size);
 		MEMDBG_PROGRAM_EXIT_CHECKS(stderr);
 		error();
 	}
diff --git a/src/memory.h b/src/memory.h
index 4f95142..a45bf9b 100644
--- a/src/memory.h
+++ b/src/memory.h
@@ -1,6 +1,6 @@
 /*
  * This file is part of John the Ripper password cracker,
- * Copyright (c) 1996-98,2003,2010-2012 by Solar Designer
+ * Copyright (c) 1996-98,2003,2010-2012,2016 by Solar Designer
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted.
@@ -72,7 +72,8 @@
 extern unsigned int mem_saving_level;
 
 /*
- * Allocates size bytes and returns a pointer to the allocated memory.
+ * Allocates size bytes and returns a pointer to the allocated memory, or NULL
+ * if size is 0.
  * If an error occurs, the function does not return.
  */
 extern void *mem_alloc_func(size_t size
@@ -81,10 +82,11 @@ extern void *mem_alloc_func(size_t size
 #endif
 	);
 /*
- * this version same as mem_alloc, but initialized the memory
- * to NULL bytes, like CALLOC(3) function does
+ * Allocates nmemb*size bytes using calloc(3) and returns a pointer to the
+ * allocated memory, or NULL if nmemb or/and size are 0.
+ * If an error occurs, the function does not return.
  */
-extern void *mem_calloc_func(size_t count, size_t size
+extern void *mem_calloc_func(size_t nmemb, size_t size
 #if defined (MEMDBG_ON)
 	, char *file, int line
 #endif
diff --git a/src/options.c b/src/options.c
index 33c7fc2..ef4e48b 100644
--- a/src/options.c
+++ b/src/options.c
@@ -1,6 +1,6 @@
 /*
  * This file is part of John the Ripper password cracker,
- * Copyright (c) 1996-2015 by Solar Designer
+ * Copyright (c) 1996-2016 by Solar Designer
  *
  * ...with changes in the jumbo patch, by JimF and magnum (and various others?)
  *
@@ -296,7 +296,7 @@ static struct opt_entry opt_list[] = {
 
 #define JOHN_USAGE	  \
 "John the Ripper " JTR_GIT_VERSION _MP_VERSION DEBUG_STRING MEMDBG_STRING ASAN_STRING UBSAN_STRING " [" JOHN_BLD "]\n" \
-"Copyright (c) 1996-2015 by " JOHN_COPYRIGHT "\n" \
+"Copyright (c) 1996-2016 by " JOHN_COPYRIGHT "\n" \
 "Homepage: http://www.openwall.com/john/\n" \
 "\n" \
 "Usage: %s [OPTIONS] [PASSWORD-FILES]\n" \
diff --git a/src/params.h b/src/params.h
index 65963bf..e17ddf6 100644
--- a/src/params.h
+++ b/src/params.h
@@ -1,6 +1,6 @@
 /*
  * This file is part of John the Ripper password cracker,
- * Copyright (c) 1996-2015 by Solar Designer
+ * Copyright (c) 1996-2016 by Solar Designer
  *
  * ...with changes in the jumbo patch, by various authors
  *
@@ -26,7 +26,7 @@
 /*
  * John's version number.
  */
-#define JOHN_VERSION			"1.8.0.6-jumbo-1-bleeding"
+#define JOHN_VERSION			"1.8.0.7-jumbo-1-bleeding"
 
 /*
  * Define this for release tarballs after updating the string above.
@@ -305,7 +305,7 @@ extern unsigned int password_hash_thresholds[PASSWORD_HASH_SIZES];
  * How many bitmap entries should the cracker prefetch at once.  Set this to 0
  * to disable prefetching.
  */
-#ifdef __SSE2__
+#ifdef __SSE__
 #define CRK_PREFETCH			64
 #else
 #define CRK_PREFETCH			0
diff --git a/src/rules.c b/src/rules.c
index d0b3905..faa06b3 100644
--- a/src/rules.c
+++ b/src/rules.c
@@ -1,6 +1,6 @@
 /*
  * This file is part of John the Ripper password cracker,
- * Copyright (c) 1996-99,2003,2005,2009,2010,2012 by Solar Designer
+ * Copyright (c) 1996-99,2003,2005,2009,2010,2015 by Solar Designer
  *
  * With heavy changes in Jumbo, by JimF and magnum
  */

Powered by blists - more mailing lists

Your e-mail address:

Powered by Openwall GNU/*/Linux - Powered by OpenVZ