Fix GH-11476: crash with count_demerits negative-size-param

Co-authored-by: Ilija Tovilo <[email protected]>
Co-authored-by: Niels Dossche <[email protected]>

Author: 3 people

ext/mbstring/mbstring.c

@@ -3083,8 +3083,9 @@ static size_t count_demerits(struct candidate *array, size_t length, bool strict
 						if (strict) {
 							/* This candidate encoding is not valid, eliminate it from consideration */
 							length--;
-							if (length == 0) {
-								return 0;
+							if (i == length) {
+								/* The eliminated candidate was the last valid one in the list */
+								goto next_iteration;
 							}
 							memmove(&array[i], &array[i+1], (length - i) * sizeof(struct candidate));
 							goto try_next_encoding;
@@ -3100,6 +3101,7 @@ static size_t count_demerits(struct candidate *array, size_t length, bool strict
 				}
 			}
 		}
+next_iteration: ;
 	}
 
 	for (size_t i = 0; i < length; i++) {

ext/mbstring/tests/gh11476.phpt

@@ -0,0 +1,12 @@
+--TEST--
+GH-11476: count_demerits negative-size-param
+--EXTENSIONS--
+mbstring
+--FILE--
+<?php
+$str = str_repeat('a', 250) . chr(246) . str_repeat('a', 9) . chr(246) . str_repeat('a', 6);
+$detectedEncoding = mb_detect_encoding($str, mb_list_encodings(), true);
+var_dump($detectedEncoding);
+?>
+--EXPECT--
+string(12) "Windows-1252"