From 386f7046f5e6f39f17281f2c0a347c50f3be9139 Mon Sep 17 00:00:00 2001 From: Karim Bahgat Date: Fri, 18 Aug 2017 10:23:11 +0200 Subject: [PATCH 1/4] More lenient reading and writing for ints and floats Previously, int fields only accepted ints, and float fields only accepted floats. Now, these can be specified as strings, and will try to force convert to the correct type. Force conversion is done only when needed, to avoid large ints getting corrupted by converting to float and back to int. Fixes #99 and #108 --- shapefile.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/shapefile.py b/shapefile.py index 3724ddb..3e1d41d 100644 --- a/shapefile.py +++ b/shapefile.py @@ -3,11 +3,11 @@ Provides read and write support for ESRI Shapefiles. author: jlawheadgeospatialpython.com date: 2017/04/29 -version: 1.2.11 +version: 1.2.12-dev Compatible with Python versions 2.7-3.x """ -__version__ = "1.2.11" +__version__ = "1.2.12-dev" from struct import pack, unpack, calcsize, error, Struct import os @@ -522,11 +522,19 @@ def __record(self): #not parseable as float, set to None value = None else: + # force to int try: - value = int(value) + # first try to force directly to int. + # forcing a large int to float and back to int + # will lose information and result in wrong nr. + value = int(value) except ValueError: - #not parseable as int, set to None - value = None + # forcing directly to int failed, so was probably a float. + try: + value = int(float(value)) + except ValueError: + #not parseable as int, set to None + value = None elif typ == 'D': # date: 8 bytes - date stored as a string in the format YYYYMMDD. if value.count(b('0')) == len(value): # QGIS NULL is all '0' chars @@ -945,8 +953,18 @@ def __dbfRecords(self): if value in MISSING: value = str("*"*size) # QGIS NULL elif not deci: + # force to int + try: + # first try to force directly to int. + # forcing a large int to float and back to int + # will lose information and result in wrong nr. + value = int(value) + except ValueError: + # forcing directly to int failed, so was probably a float. + value = int(float(value)) value = format(value, "d")[:size].rjust(size) # caps the size if exceeds the field size else: + value = float(value) value = format(value, ".%sf"%deci)[:size].rjust(size) # caps the size if exceeds the field size elif fieldType == "D": # date: 8 bytes - date stored as a string in the format YYYYMMDD. From 10c61763cce6def80e91968d79e6d09823edab8d Mon Sep 17 00:00:00 2001 From: Karim Bahgat Date: Thu, 24 Aug 2017 16:54:09 +0200 Subject: [PATCH 2/4] Update to new bugfix version --- changelog.txt | 6 ++++++ setup.py | 2 +- shapefile.py | 6 +++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/changelog.txt b/changelog.txt index 9ff0a00..f7785ee 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,9 @@ +VERSION 1.2.12 + +2017-08-24 Karim Bahgat + * Fixed errors caused by strict value type checking, as introduced in v1.2.11. Now more lenient by attempting force conversion of values to match the field type. + * Allow reading file-like objects without seek method (such as ZipFile or urllib.urlopen). + VERSION 1.2.11 2017-04-29 Karim Bahgat diff --git a/setup.py b/setup.py index 9acd4f7..09a667b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup setup(name='pyshp', - version='1.2.11', + version='1.2.12', description='Pure Python read/write support for ESRI Shapefile format', long_description=open('README.md').read(), author='Joel Lawhead', diff --git a/shapefile.py b/shapefile.py index 3e1d41d..dbf846a 100644 --- a/shapefile.py +++ b/shapefile.py @@ -2,12 +2,12 @@ shapefile.py Provides read and write support for ESRI Shapefiles. author: jlawheadgeospatialpython.com -date: 2017/04/29 -version: 1.2.12-dev +date: 2017/08/24 +version: 1.2.12 Compatible with Python versions 2.7-3.x """ -__version__ = "1.2.12-dev" +__version__ = "1.2.12" from struct import pack, unpack, calcsize, error, Struct import os From a53f022d8f77f902e622dec1e4483c601aa42817 Mon Sep 17 00:00:00 2001 From: Jason Moujaes Date: Sat, 25 Aug 2018 11:34:08 -0700 Subject: [PATCH 3/4] Edit docs to clarify that Points dont have a bounding box attribute 'bbox'. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 380ffd2..96ee555 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ method. >>> len(list(sf.iterShapes())) 663 -Each shape record contains the following attributes: +Each shape record (except Points) contain the following attributes. Records of shapeType Point do not have a bounding box 'bbox'. >>> for name in dir(shapes[3]): From fc630bb5de025cae1886d7fd415a3372e28d47d2 Mon Sep 17 00:00:00 2001 From: Karim Bahgat Date: Sat, 8 Sep 2018 16:09:20 -0400 Subject: [PATCH 4/4] Fix text character truncation in py3 + bump to next version - Fixes issue in Py3 when converting text characters to byte strings, but in Py3 converts to unicode instead, because uses the Py2 specific str() function, instead of the version neutral b(). When the text contains non-ascii 2-byte unicode values this results in truncating the unicode length instead of the byte length, and thus results in incorrectly padded byte lengths and data values ending up in the wrong field/column. See #157, and also #148. - Also bump to next version. --- shapefile.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shapefile.py b/shapefile.py index dbf846a..8b0fff4 100644 --- a/shapefile.py +++ b/shapefile.py @@ -2,12 +2,12 @@ shapefile.py Provides read and write support for ESRI Shapefiles. author: jlawheadgeospatialpython.com -date: 2017/08/24 -version: 1.2.12 +date: 2018/09/8 +version: 1.2.13 Compatible with Python versions 2.7-3.x """ -__version__ = "1.2.12" +__version__ = "1.2.13" from struct import pack, unpack, calcsize, error, Struct import os @@ -989,8 +989,8 @@ def __dbfRecords(self): else: value = b(' ') # unknown is set to space else: - # anything else is forced to string - value = str(value)[:size].ljust(size) + # anything else is forced to byte string + value = b(value)[:size].ljust(size) if len(value) != size: raise ShapefileException( "Shapefile Writer unable to pack incorrect sized value"