From 7bfa10018ecf5efe54b4a699bb684d31468c8b42 Mon Sep 17 00:00:00 2001
From: Binh-Minh Ribler <bmribler@hdfgroup.org>
Date: Mon, 26 Oct 2020 08:36:27 -0500
Subject: [PATCH] Fix HDFFV-10590

Description
    This is to fix the CVE issue CVE-2018-17432.
    h5repack produced a segfault on a corrupted file.  This fix modified
the
    dataspace encode and decode functions per Quincey's suggestion to
prevent
    the segfault.  h5repack only failed for the corrupted file now.
Platforms tested:
    Linux/64 (jelly)

---
 src/H5Osdspace.c      |  23 +-
 1 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c
index 627ea190a3..6e34960d87 100644
--- a/src/H5Osdspace.c
+++ b/src/H5Osdspace.c
@@ -143,8 +143,11 @@ H5O_sdspace_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED
     flags = *p++;
 
     /* Get or determine the type of the extent */
-    if(version >= H5O_SDSPACE_VERSION_2)
+    if(version >= H5O_SDSPACE_VERSION_2) {
         sdim->type = (H5S_class_t)*p++;
+        if(sdim->type != H5S_SIMPLE && sdim->rank > 0)
+            HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid rank for scalar or NULL dataspace")
+    } /* end if */
     else {
         /* Set the dataspace type to be simple or scalar as appropriate */
         if(sdim->rank > 0)
@@ -267,14 +270,16 @@ H5O_sdspace_encode(H5F_t *f, uint8_t *p, const void *_mesg)
         *p++ = 0; /*reserved*/
     } /* end else */
 
-    /* Current & maximum dimensions */
-    if(sdim->rank > 0) {
-        for(u = 0; u < sdim->rank; u++)
-            H5F_ENCODE_LENGTH(f, p, sdim->size[u]);
-        if(flags & H5S_VALID_MAX) {
-            for(u = 0; u < sdim->rank; u++)
-                H5F_ENCODE_LENGTH(f, p, sdim->max[u]);
-        } /* end if */
+    /* Encode dataspace dimensions for simple dataspaces */
+    if(H5S_SIMPLE == sdim->type) {
+        /* Encode current & maximum dimensions */
+       if(sdim->rank > 0) {
+           for(u = 0; u < sdim->rank; u++)
+               H5F_ENCODE_LENGTH(f, p, sdim->size[u]);
+           if(flags & H5S_VALID_MAX)
+              for(u = 0; u < sdim->rank; u++)
+                  H5F_ENCODE_LENGTH(f, p, sdim->max[u]);
+       } /* end if */
     } /* end if */
 
     FUNC_LEAVE_NOAPI(SUCCEED)
