epan: Fix the end offsets for hex string items

hex_str_to_bytes_encoding() consumes pairs of hex digits (and
optional separator) to turn into bytes. It can return a pointer
to the character after the last digit consumed. Don't advance
the end pointer after a single unpaired digit that is not consumed
as part of the hex string returned.

tvb_get_string_bytes() can pass back the end offset. If conversion
fails, return the initial offset instead of zero to make repeated
calls easier in cases where the full length is not decoded due to
errors.

Relatedly, no dissector currently uses this return value, because
it's not useful currently.

epan/strutil.c

@@ -339,16 +339,15 @@ hex_str_to_bytes_encoding(const gchar *hex_str, GByteArray *bytes, const gchar *
                 if (fail_if_partial) retval = FALSE;
                 break;
             }
-            ++end;
 
-            d = str_to_nibble[(guchar)*end];
+            d = str_to_nibble[(guchar)*(end+1)];
             if (d < 0) {
                 if (fail_if_partial) retval = FALSE;
                 break;
             }
             val = ((guint8)c * 16) + d;
             g_byte_array_append(bytes, &val, 1);
-            ++end;
+            end += 2;
 
             /* check for separator and peek at next char to make sure we should keep going */
             if (sep > 0 && *end == sep && str_to_nibble[(guchar)*(end+1)] > -1) {

epan/tvbuff.c

@@ -1748,7 +1748,7 @@ tvb_get_string_bytes(tvbuff_t *tvb, const gint offset, const gint length,
 	ptr = (gchar*) tvb_get_raw_string(NULL, tvb, offset, length);
 	begin = ptr;
 
-	if (endoff) *endoff = 0;
+	if (endoff) *endoff = offset;
 
 	while (*begin == ' ') begin++;