make hashes always the size of pointers; introduce Py_hash_t #9778

Author: benjaminp

Doc/c-api/object.rst

@@ -247,15 +247,20 @@ is considered sufficient for this determination.
    *NULL* on failure.
 
 
-.. c:function:: long PyObject_Hash(PyObject *o)
+.. c:function:: Py_hash_t PyObject_Hash(PyObject *o)
 
    .. index:: builtin: hash
 
    Compute and return the hash value of an object *o*.  On failure, return ``-1``.
    This is the equivalent of the Python expression ``hash(o)``.
 
+   .. versionchanged:: 3.2
 
-.. c:function:: long PyObject_HashNotImplemented(PyObject *o)
+      The return type is now Py_hash_t.  This is a signed integer the same size
+      as Py_ssize_t.
+
+
+.. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o)
 
    Set a :exc:`TypeError` indicating that ``type(o)`` is not hashable and return ``-1``.
    This function receives special treatment when stored in a ``tp_hash`` slot,

Doc/c-api/typeobj.rst

@@ -306,10 +306,10 @@ type objects) *must* have the :attr:`ob_size` field.
    An optional pointer to a function that implements the built-in function
    :func:`hash`.
 
-   The signature is the same as for :c:func:`PyObject_Hash`; it must return a C
-   long.  The value ``-1`` should not be returned as a normal return value; when an
-   error occurs during the computation of the hash value, the function should set
-   an exception and return ``-1``.
+   The signature is the same as for :c:func:`PyObject_Hash`; it must return a
+   value of the type Py_hash_t.  The value ``-1`` should not be returned as a
+   normal return value; when an error occurs during the computation of the hash
+   value, the function should set an exception and return ``-1``.
 
    This field can be set explicitly to :c:func:`PyObject_HashNotImplemented` to
    block inheritance of the hash method from a parent type. This is interpreted

Include/bytesobject.h

@@ -29,7 +29,7 @@ functions should be applied to nil objects.
 
 typedef struct {
     PyObject_VAR_HEAD
-    long ob_shash;
+    Py_hash_t ob_shash;
     char ob_sval[1];
 
     /* Invariants:

Include/datetime.h

@@ -34,7 +34,7 @@ extern "C" {
 typedef struct
 {
     PyObject_HEAD
-    long hashcode;              /* -1 when unknown */
+    Py_hash_t hashcode;         /* -1 when unknown */
     int days;                   /* -MAX_DELTA_DAYS <= days <= MAX_DELTA_DAYS */
     int seconds;                /* 0 <= seconds < 24*3600 is invariant */
     int microseconds;           /* 0 <= microseconds < 1000000 is invariant */
@@ -51,7 +51,7 @@ typedef struct
  */
 #define _PyTZINFO_HEAD          \
     PyObject_HEAD               \
-    long hashcode;              \
+    Py_hash_t hashcode;         \
     char hastzinfo;             /* boolean flag */
 
 /* No _PyDateTime_BaseTZInfo is allocated; it's just to have something

Include/dictobject.h

@@ -48,11 +48,8 @@ meaning otherwise.
 #define PyDict_MINSIZE 8
 
 typedef struct {
-    /* Cached hash code of me_key.  Note that hash codes are C longs.
-     * We have to use Py_ssize_t instead because dict_popitem() abuses
-     * me_hash to hold a search finger.
-     */
-    Py_ssize_t me_hash;
+    /* Cached hash code of me_key. */
+    Py_hash_t me_hash;
     PyObject *me_key;
     PyObject *me_value;
 } PyDictEntry;
@@ -84,7 +81,7 @@ struct _dictobject {
      * setitem calls.
      */
     PyDictEntry *ma_table;
-    PyDictEntry *(*ma_lookup)(PyDictObject *mp, PyObject *key, long hash);
+    PyDictEntry *(*ma_lookup)(PyDictObject *mp, PyObject *key, Py_hash_t hash);
     PyDictEntry ma_smalltable[PyDict_MINSIZE];
 };
 
@@ -116,14 +113,14 @@ PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
 PyAPI_FUNC(int) PyDict_Next(
     PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value);
 PyAPI_FUNC(int) _PyDict_Next(
-    PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, long *hash);
+    PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash);
 PyAPI_FUNC(PyObject *) PyDict_Keys(PyObject *mp);
 PyAPI_FUNC(PyObject *) PyDict_Values(PyObject *mp);
 PyAPI_FUNC(PyObject *) PyDict_Items(PyObject *mp);
 PyAPI_FUNC(Py_ssize_t) PyDict_Size(PyObject *mp);
 PyAPI_FUNC(PyObject *) PyDict_Copy(PyObject *mp);
 PyAPI_FUNC(int) PyDict_Contains(PyObject *mp, PyObject *key);
-PyAPI_FUNC(int) _PyDict_Contains(PyObject *mp, PyObject *key, long hash);
+PyAPI_FUNC(int) _PyDict_Contains(PyObject *mp, PyObject *key, Py_hash_t hash);
 PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused);
 PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp);
 PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp);

Include/object.h

@@ -275,7 +275,7 @@ typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
 typedef int (*setattrfunc)(PyObject *, char *, PyObject *);
 typedef int (*setattrofunc)(PyObject *, PyObject *, PyObject *);
 typedef PyObject *(*reprfunc)(PyObject *);
-typedef long (*hashfunc)(PyObject *);
+typedef Py_hash_t (*hashfunc)(PyObject *);
 typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
 typedef PyObject *(*getiterfunc) (PyObject *);
 typedef PyObject *(*iternextfunc) (PyObject *);
@@ -440,8 +440,8 @@ PyAPI_FUNC(PyObject *) _PyObject_NextNotImplemented(PyObject *);
 PyAPI_FUNC(PyObject *) PyObject_GenericGetAttr(PyObject *, PyObject *);
 PyAPI_FUNC(int) PyObject_GenericSetAttr(PyObject *,
                                               PyObject *, PyObject *);
-PyAPI_FUNC(long) PyObject_Hash(PyObject *);
-PyAPI_FUNC(long) PyObject_HashNotImplemented(PyObject *);
+PyAPI_FUNC(Py_hash_t) PyObject_Hash(PyObject *);
+PyAPI_FUNC(Py_hash_t) PyObject_HashNotImplemented(PyObject *);
 PyAPI_FUNC(int) PyObject_IsTrue(PyObject *);
 PyAPI_FUNC(int) PyObject_Not(PyObject *);
 PyAPI_FUNC(int) PyCallable_Check(PyObject *);
@@ -470,8 +470,8 @@ PyAPI_FUNC(int) Py_ReprEnter(PyObject *);
 PyAPI_FUNC(void) Py_ReprLeave(PyObject *);
 
 /* Helpers for hash functions */
-PyAPI_FUNC(long) _Py_HashDouble(double);
-PyAPI_FUNC(long) _Py_HashPointer(void*);
+PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double);
+PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*);
 
 /* Helper for passing objects to printf and the like */
 #define PyObject_REPR(obj) _PyUnicode_AsString(PyObject_Repr(obj))

Include/pyport.h

@@ -130,7 +130,7 @@ Used in:  PY_LONG_LONG
    _PyHash_Double in Objects/object.c.  Numeric hashes are based on
    reduction modulo the prime 2**_PyHASH_BITS - 1. */
 
-#if SIZEOF_LONG >= 8
+#if SIZEOF_VOID_P >= 8
 #define _PyHASH_BITS 61
 #else
 #define _PyHASH_BITS 31
@@ -177,6 +177,9 @@ typedef Py_intptr_t     Py_ssize_t;
 #   error "Python needs a typedef for Py_ssize_t in pyport.h."
 #endif
 
+/* Py_hash_t is the same size as a pointer. */
+typedef Py_ssize_t Py_hash_t;
+
 /* Largest possible value of size_t.
    SIZE_MAX is part of C99, so it might be defined on some
    platforms. If it is not defined, (size_t)-1 is a portable

Include/setobject.h

@@ -22,11 +22,8 @@ no meaning otherwise.
 #define PySet_MINSIZE 8
 
 typedef struct {
-    /* Cached hash code of the key.  Note that hash codes are C longs.
-     * We have to use Py_ssize_t instead because set_pop() abuses
-     * the hash field to hold a search finger.
-     */
-    Py_ssize_t hash;
+    /* Cached hash code of the key. */
+    Py_hash_t hash;
     PyObject *key;
 } setentry;
 
@@ -53,10 +50,10 @@ struct _setobject {
      * saves repeated runtime null-tests.
      */
     setentry *table;
-    setentry *(*lookup)(PySetObject *so, PyObject *key, long hash);
+    setentry *(*lookup)(PySetObject *so, PyObject *key, Py_hash_t hash);
     setentry smalltable[PySet_MINSIZE];
 
-    long hash;                  /* only used by frozenset objects */
+    Py_hash_t hash;                  /* only used by frozenset objects */
     PyObject *weakreflist;      /* List of weak references */
 };
 
@@ -93,7 +90,7 @@ PyAPI_FUNC(int) PySet_Clear(PyObject *set);
 PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key);
 PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key);
 PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key);
-PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, long *hash);
+PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash);
 PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set);
 PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
 

Include/unicodeobject.h

@@ -372,7 +372,7 @@ typedef struct {
     PyObject_HEAD
     Py_ssize_t length;          /* Length of raw Unicode data in buffer */
     Py_UNICODE *str;            /* Raw Unicode buffer */
-    long hash;                  /* Hash value; -1 if not set */
+    Py_hash_t hash;             /* Hash value; -1 if not set */
     int state;                  /* != 0 if interned. In this case the two
                                  * references from the dictionary to this object
                                  * are *not* counted in ob_refcnt. */

Include/weakrefobject.h

@@ -27,7 +27,7 @@ struct _PyWeakReference {
     /* A cache for wr_object's hash code.  As usual for hashes, this is -1
      * if the hash code isn't known yet.
      */
-    long hash;
+    Py_hash_t hash;
 
     /* If wr_object is weakly referenced, wr_object has a doubly-linked NULL-
      * terminated list of weak references to it.  These are the list pointers.

Misc/NEWS

@@ -93,6 +93,12 @@ Extensions
 
 - The Unicode database was updated to 6.0.0.
 
+C-API
+-----
+
+- Issue #9778: Hash values are now always the size of pointers. A new Py_hash_t
+  type has been introduced.
+
 Tools/Demos
 -----------
 

Modules/_datetimemodule.c

@@ -1843,7 +1843,7 @@ delta_richcompare(PyObject *self, PyObject *other, int op)
 
 static PyObject *delta_getstate(PyDateTime_Delta *self);
 
-static long
+static Py_hash_t
 delta_hash(PyDateTime_Delta *self)
 {
     if (self->hashcode == -1) {
@@ -2777,11 +2777,11 @@ date_replace(PyDateTime_Date *self, PyObject *args, PyObject *kw)
 /*
     Borrowed from stringobject.c, originally it was string_hash()
 */
-static long
+static Py_hash_t
 generic_hash(unsigned char *data, int len)
 {
     register unsigned char *p;
-    register long x;
+    register Py_hash_t x;
 
     p = (unsigned char *) data;
     x = *p << 7;
@@ -2797,7 +2797,7 @@ generic_hash(unsigned char *data, int len)
 
 static PyObject *date_getstate(PyDateTime_Date *self);
 
-static long
+static Py_hash_t
 date_hash(PyDateTime_Date *self)
 {
     if (self->hashcode == -1)
@@ -3246,7 +3246,7 @@ timezone_richcompare(PyDateTime_TimeZone *self,
     return delta_richcompare(self->offset, other->offset, op);
 }
 
-static long
+static Py_hash_t
 timezone_hash(PyDateTime_TimeZone *self)
 {
     return delta_hash((PyDateTime_Delta *)self->offset);
@@ -3751,7 +3751,7 @@ time_richcompare(PyObject *self, PyObject *other, int op)
     return result;
 }
 
-static long
+static Py_hash_t
 time_hash(PyDateTime_Time *self)
 {
     if (self->hashcode == -1) {
@@ -4640,7 +4640,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op)
     return result;
 }
 
-static long
+static Py_hash_t
 datetime_hash(PyDateTime_DateTime *self)
 {
     if (self->hashcode == -1) {

Modules/_pickle.c

@@ -486,7 +486,7 @@ _PyMemoTable_Lookup(PyMemoTable *self, PyObject *key)
     size_t mask = (size_t)self->mt_mask;
     PyMemoEntry *table = self->mt_table;
     PyMemoEntry *entry;
-    long hash = (long)key >> 3;
+    Py_hash_t hash = (Py_hash_t)key >> 3;
 
     i = hash & mask;
     entry = &table[i];

Objects/bytesobject.c

@@ -868,12 +868,12 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op)
     return result;
 }
 
-static long
+static Py_hash_t
 bytes_hash(PyBytesObject *a)
 {
     register Py_ssize_t len;
     register unsigned char *p;
-    register long x;
+    register Py_hash_t x;
 
     if (a->ob_shash != -1)
         return a->ob_shash;

Objects/classobject.c

@@ -263,10 +263,10 @@ method_repr(PyMethodObject *a)
     return result;
 }
 
-static long
+static Py_hash_t
 method_hash(PyMethodObject *a)
 {
-    long x, y;
+    Py_hash_t x, y;
     if (a->im_self == NULL)
         x = PyObject_Hash(Py_None);
     else

Objects/codeobject.c

@@ -417,10 +417,10 @@ code_richcompare(PyObject *self, PyObject *other, int op)
     return res;
 }
 
-static long
+static Py_hash_t
 code_hash(PyCodeObject *co)
 {
-    long h, h0, h1, h2, h3, h4, h5, h6;
+    Py_hash_t h, h0, h1, h2, h3, h4, h5, h6;
     h0 = PyObject_Hash(co->co_name);
     if (h0 == -1) return -1;
     h1 = PyObject_Hash(co->co_code);

Objects/complexobject.c

@@ -394,7 +394,7 @@ complex_repr(PyComplexObject *v)
     return complex_format(v, 0, 'r');
 }
 
-static long
+static Py_hash_t
 complex_hash(PyComplexObject *v)
 {
     unsigned long hashreal, hashimag, combined;
@@ -413,7 +413,7 @@ complex_hash(PyComplexObject *v)
     combined = hashreal + _PyHASH_IMAG * hashimag;
     if (combined == (unsigned long)-1)
         combined = (unsigned long)-2;
-    return (long)combined;
+    return (Py_hash_t)combined;
 }
 
 /* This macro may return! */

Objects/descrobject.c

@@ -916,10 +916,10 @@ wrapper_richcompare(PyObject *a, PyObject *b, int op)
     return v;
 }
 
-static long
+static Py_hash_t
 wrapper_hash(wrapperobject *wp)
 {
-    int x, y;
+    Py_hash_t x, y;
     x = _Py_HashPointer(wp->descr);
     if (x == -1)
         return -1;