Fix memory leak when encoding check fails

zlib_create_dictionary_string() allocates memory, so we can leak memory
if there's an early exit before the assignment to the return value.
Solve this by moving all validation upwards.

Closes GH-17788.

Author: nielsdos

NEWS

@@ -41,6 +41,7 @@ PHP                                                                        NEWS
 - Zlib:
   . Fixed bug GH-17745 (zlib extension incorrectly handles object arguments).
     (nielsdos)
+  . Fix memory leak when encoding check fails. (nielsdos)
 
 13 Feb 2025, PHP 8.3.17
 

ext/zlib/tests/leak_invalid_encoding_with_dict.phpt

@@ -0,0 +1,20 @@
+--TEST--
+Memory leak when passing a dictionary with invalid encoding
+--EXTENSIONS--
+zlib
+--FILE--
+<?php
+try {
+	inflate_init(123456, ["dictionary" => "dict"]);
+} catch (ValueError $e) {
+	echo $e->getMessage(), "\n";
+}
+try {
+	deflate_init(123456, ["dictionary" => "dict"]);
+} catch (ValueError $e) {
+	echo $e->getMessage(), "\n";
+}
+?>
+--EXPECT--
+Encoding mode must be ZLIB_ENCODING_RAW, ZLIB_ENCODING_GZIP or ZLIB_ENCODING_DEFLATE
+deflate_init(): Argument #1 ($encoding) must be one of ZLIB_ENCODING_RAW, ZLIB_ENCODING_GZIP, or ZLIB_ENCODING_DEFLATE

ext/zlib/zlib.c

@@ -879,10 +879,6 @@ PHP_FUNCTION(inflate_init)
 		RETURN_THROWS();
 	}
 
-	if (!zlib_create_dictionary_string(options, &dict, &dictlen)) {
-		RETURN_THROWS();
-	}
-
 	switch (encoding) {
 		case PHP_ZLIB_ENCODING_RAW:
 		case PHP_ZLIB_ENCODING_GZIP:
@@ -893,6 +889,10 @@ PHP_FUNCTION(inflate_init)
 			RETURN_THROWS();
 	}
 
+	if (!zlib_create_dictionary_string(options, &dict, &dictlen)) {
+		RETURN_THROWS();
+	}
+
 	object_init_ex(return_value, inflate_context_ce);
 	ctx = Z_INFLATE_CONTEXT_P(return_value);
 
@@ -1132,10 +1132,6 @@ PHP_FUNCTION(deflate_init)
 			RETURN_THROWS();
 	}
 
-	if (!zlib_create_dictionary_string(options, &dict, &dictlen)) {
-		RETURN_THROWS();
-	}
-
 	switch (encoding) {
 		case PHP_ZLIB_ENCODING_RAW:
 		case PHP_ZLIB_ENCODING_GZIP:
@@ -1146,6 +1142,10 @@ PHP_FUNCTION(deflate_init)
 			RETURN_THROWS();
 	}
 
+	if (!zlib_create_dictionary_string(options, &dict, &dictlen)) {
+		RETURN_THROWS();
+	}
+
 	object_init_ex(return_value, deflate_context_ce);
 	ctx = Z_DEFLATE_CONTEXT_P(return_value);