# HG changeset patch
# User Franziskus Kiefer <franziskuskiefer@gmail.com>
# Date 1489748381 -3600
# Node ID 77a5bb81dbaac5b03266a64ff981c156b61c8931
# Parent  da15c12097edbe876620ed5da8378ee3269caea8
Bug 1344380 - gtests for b64 bug and some fixes, r=ttaubert

Differential Revision: https://nss-review.dev.mozaws.net/D256#inline-2146

Origin: backport, https://hg.mozilla.org/projects/nss/rev/77a5bb81dbaa
--- nss-3.26.orig/nss/lib/util/nssb64d.c
+++ nss-3.26/nss/lib/util/nssb64d.c
@@ -373,7 +373,7 @@ pl_base64_decode_flush (PLBase64Decoder
 static PRUint32
 PL_Base64MaxDecodedLength (PRUint32 size)
 {
-    return ((size * 3) / 4);
+    return size * 0.75;
 }
 
 
--- a/nss/lib/util/nssb64e.c
+++ b/nss/lib/util/nssb64e.c
@@ -285,20 +285,29 @@ PL_Base64MaxEncodedLength (PRUint32 size
 {
     PRUint32 tokens, tokens_per_line, full_lines, line_break_chars, remainder;
 
+    /* This is the maximum length we support. */
+    if (size > 0x3fffffff) {
+        return 0;
+    }
+
+
     tokens = (size + 2) / 3;
 
-    if (line_length == 0)
+    if (line_length == 0) {
 	return tokens * 4;
+    }
 
-    if (line_length < 4)	/* too small! */
+    if (line_length < 4) { /* too small! */
 	line_length = 4;
+    }
 
     tokens_per_line = line_length / 4;
     full_lines = tokens / tokens_per_line;
     remainder = (tokens - (full_lines * tokens_per_line)) * 4;
     line_break_chars = full_lines * 2;
-    if (remainder == 0)
+    if (remainder == 0) {
 	line_break_chars -= 2;
+    }
 
     return (full_lines * tokens_per_line * 4) + line_break_chars + remainder;
 }
@@ -454,13 +463,18 @@ PL_Base64EncodeBuffer (const unsigned ch
     PRStatus status;
 
     PR_ASSERT(srclen > 0);
-    if (srclen == 0)
+    if (srclen == 0) {
 	return dest;
+    }
 
     /*
      * How much space could we possibly need for encoding this input?
      */
     need_length = PL_Base64MaxEncodedLength (srclen, line_length);
+    if (need_length == 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return NULL;
+    }
 
     /*
      * Make sure we have at least that much, if output buffer provided.
@@ -643,6 +657,10 @@ NSSBase64_EncodeItem (PLArenaPool *arena
     }
 
     max_out_len = PL_Base64MaxEncodedLength (inItem->len, 64);
+    if (max_out_len == 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return NULL;
+    }
 
     if (arenaOpt != NULL)
 	mark = PORT_ArenaMark (arenaOpt);