diff --git a/.flake8 b/.flake8
new file mode 100644
index 00000000..b33811f1
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501, W503
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..80224f9e
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+tests/samples/* linguist-vendored
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
new file mode 100644
index 00000000..23f16106
--- /dev/null
+++ b/.github/workflows/python-package.yml
@@ -0,0 +1,40 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+ push:
+ branches: [ "master" ]
+ pull_request:
+ branches: [ "master" ]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install flake8 pytest
+ if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+ - name: Lint with flake8
+ run: |
+ # stop the build if there are Python syntax errors or undefined names
+ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+ # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+ flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ - name: Test with pytest
+ run: |
+ pytest
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
new file mode 100644
index 00000000..bdaab28a
--- /dev/null
+++ b/.github/workflows/python-publish.yml
@@ -0,0 +1,39 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+ release:
+ types: [published]
+
+permissions:
+ contents: read
+
+jobs:
+ deploy:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: '3.x'
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install build
+ - name: Build package
+ run: python -m build
+ - name: Publish package
+ uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+ with:
+ user: __token__
+ password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.gitignore b/.gitignore
index 0d20b648..b532e65e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,18 @@
*.pyc
+__pycache__
+*.egg-info
+/build
+/dist
+/bin
+/include
+/lib
+/local
+/man
+nosetests.xml
+.coverage
+.tox
+.idea
+.cache
+/.noseids
+/.venv
+/poetry.lock
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 00000000..ea56f519
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,29 @@
+language: python
+os: linux
+cache: pip
+
+matrix:
+ include:
+ - name: "Python 3.8 on Linux"
+ dist: xenial
+ python: 3.8
+ - name: "Python 3.9 Nightly on Linux"
+ dist: bionic
+ python: nightly
+ - name: "Pypy 3 on Linux"
+ python: pypy3
+ allow_failures:
+ - python: nightly
+ - python: pypy3
+ - os: osx
+
+install:
+ - if [ $PIP ]; then true; else PIP=pip3; fi
+ - travis_retry $PIP install -U pip wheel tox-travis pytest-cov codecov
+ - travis_retry $PIP install -U -r requirements.txt -e ".[test]"
+
+script:
+ - tox
+
+after_success:
+ - codecov
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..261eeb9e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..9caf08a5
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,73 @@
+# Makefile to help automate tasks
+WD := $(shell pwd)
+PY := .venv/bin/python
+PIP := .venv/bin/pip
+PEP8 := .venv/bin/pep8
+NOSE := .venv/bin/nosetests
+TWINE := .venv/bin/twine
+
+# ###########
+# Tests rule!
+# ###########
+.PHONY: test
+test: venv develop $(NOSE)
+ $(NOSE) --with-id -s tests
+
+$(NOSE): setup
+
+# #######
+# INSTALL
+# #######
+.PHONY: all
+all: setup develop
+
+venv: .venv/bin/python
+
+setup: venv
+ $(PIP) install -r requirements-dev.txt | grep -v "already satisfied" || true
+
+.venv/bin/python:
+ test -d .venv || which python3 && python3 -m venv .venv || virtualenv .venv
+
+.PHONY: clean
+clean:
+ rm -rf .venv
+
+develop: .venv/lib/python*/site-packages/readability-lxml.egg-link
+
+.venv/lib/python*/site-packages/readability-lxml.egg-link:
+ $(PY) setup.py develop
+
+
+# ###########
+# Development
+# ###########
+.PHONY: clean_all
+clean_all: clean_venv
+
+.PHONY: build
+build:
+ poetry build
+
+# ###########
+# Deploy
+# ###########
+.PHONY: dist
+dist:
+ $(PY) -m pip install wheel
+ $(PY) setup.py sdist bdist_wheel
+ $(TWINE) check dist/*
+
+.PHONY: upload
+upload:
+ $(TWINE) upload dist/*
+
+.PHONY: bump
+bump:
+ $(EDITOR) readability/__init__.py
+ $(eval VERSION := $(shell grep "__version__" readability/__init__.py | cut -d'"' -f2))
+ # fix first occurrence of version in pyproject.toml
+ sed -i '0,/version = ".*"/s//version = "$(VERSION)"/' pyproject.toml
+ git commit -m "Bump version to $(VERSION)" pyproject.toml readability/__init__.py
+ git tag $(VERSION)
+ git push --tags
diff --git a/README b/README
deleted file mode 100644
index f7b86b35..00000000
--- a/README
+++ /dev/null
@@ -1,23 +0,0 @@
-This code is under the Apache License 2.0. http://www.apache.org/licenses/LICENSE-2.0
-
-This is a python port of a ruby port of arc90's readability project
-
-http://lab.arc90.com/experiments/readability/
-
-Given a html document, it pulls out the main body text and cleans it up.
-
-Ruby port by starrhorne and iterationlabs
-Python port by gfxmonk
-
-This port uses BeautifulSoup for the HTML parsing. That means it can be
-a little slow, but will work on Google App Engine (unlike libxml-based
-libraries)
-
-
-**note**: I don't currently have any plans for using or improving this
-library, and it's far from perfect (slow, and almost certainly buggy).
-So if you do something cool with it or have a better tool that does
-the same job, please let me know and I can link to it from here.
-
-If you're looking for alternatives, here's the list so far:
- - http://www.minvolai.com/blog/decruft-arc90s-readability-in-python/
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..e09a515a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,67 @@
+[](https://pypi.python.org/pypi/readability-lxml)
+
+# python-readability
+
+Given an HTML document, extract and clean up the main body text and title.
+
+This is a Python port of a Ruby port of [arc90's Readability project](https://web.archive.org/web/20130519040221/http://www.readability.com/).
+
+## Installation
+
+It's easy using `pip`, just run:
+
+```bash
+$ pip install readability-lxml
+```
+
+As an alternative, you may also use conda to install, just run:
+
+```bash
+$ conda install -c conda-forge readability-lxml
+```
+
+## Usage
+
+```python
+>>> import requests
+>>> from readability import Document
+
+>>> response = requests.get('http://example.com/')
+>>> doc = Document(response.content)
+>>> doc.title()
+'Example Domain'
+
+>>> doc.summary()
+"""
\n
\n
Example Domain
\n
+
This domain is established to be used for illustrative examples in documents. You may
+use this\n domain in examples without prior coordination or asking for permission.
"""
+```
+
+## Change Log
+- 0.8.4 Better CJK support, thanks @cdhigh
+- 0.8.3.1 Support for python 3.8 - 3.13
+- 0.8.3 We can now save all images via keep_all_images=True (default is to save 1 main image), thanks @botlabsDev
+- 0.8.2 Added article author(s) (thanks @mattblaha)
+- 0.8.1 Fixed processing of non-ascii HTMLs via regexps.
+- 0.8 Replaced XHTML output with HTML5 output in summary() call.
+- 0.7.1 Support for Python 3.7 . Fixed a slowdown when processing documents with lots of spaces.
+- 0.7 Improved HTML5 tags handling. Fixed stripping unwanted HTML nodes (only first matching node was removed before).
+- 0.6 Finally a release which supports Python versions 2.6, 2.7, 3.3 - 3.6
+- 0.5 Preparing a release to support Python versions 2.6, 2.7, 3.3 and 3.4
+- 0.4 Added Videos loading and allowed more images per paragraph
+- 0.3 Added Document.encoding, positive\_keywords and negative\_keywords
+
+## Licensing
+
+This code is under [the Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0) license.
+
+## Thanks to
+
+- Latest [readability.js](https://github.com/MHordecki/readability-redux/blob/master/readability/readability.js)
+- Ruby port by starrhorne and iterationlabs
+- [Python port](https://github.com/gfxmonk/python-readability) by gfxmonk
+- [Decruft effort](https://web.archive.org/web/20110214150709/https://www.minvolai.com/blog/decruft-arc90s-readability-in-python/) to move to lxml
+- "BR to P" fix from readability.js which improves quality for smaller texts
+- Github users contributions.
diff --git a/doc/__init__.py b/doc/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/doc/source/__init__.py b/doc/source/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/doc/source/api.rst b/doc/source/api.rst
new file mode 100644
index 00000000..b0e3bbbb
--- /dev/null
+++ b/doc/source/api.rst
@@ -0,0 +1,30 @@
+Reference
+=========
+
+.. automodule:: readability
+ :members:
+ :show-inheritance:
+
+.. automodule:: readability.browser
+ :members:
+ :show-inheritance:
+
+.. automodule:: readability.cleaners
+ :members:
+ :show-inheritance:
+
+.. automodule:: readability.debug
+ :members:
+ :show-inheritance:
+
+.. automodule:: readability.encoding
+ :members:
+ :show-inheritance:
+
+.. automodule:: readability.htmls
+ :members:
+ :show-inheritance:
+
+.. automodule:: readability.readability
+ :members:
+ :show-inheritance:
diff --git a/doc/source/conf.py b/doc/source/conf.py
new file mode 100644
index 00000000..e70cf9b3
--- /dev/null
+++ b/doc/source/conf.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+#
+# readability documentation build configuration file, created by
+# sphinx-quickstart on Thu Mar 23 16:29:38 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+import readability
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ "sphinx.ext.autodoc",
+ "sphinx.ext.doctest",
+ "sphinx.ext.intersphinx",
+ "sphinx.ext.todo",
+ "myst_parser",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = [".rst", ".md"]
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "readability"
+copyright = "2020, Yuri Baburov"
+author = "Yuri Baburov"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+
+# The short X.Y version.
+version = readability.__version__
+
+# The full version, including alpha/beta/rc tags.
+release = readability.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = "en"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = [] #'_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "readabilitydoc"
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [(master_doc, "readability.tex", "Readability Documentation", "Yuri Baburov", "manual")]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "readability", "readability Documentation", [author], 1)]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (
+ master_doc,
+ "readability",
+ "Readability Documentation",
+ author,
+ "readability",
+ "One line description of project.",
+ "Miscellaneous",
+ )
+]
+
+
+intersphinx_mapping = {
+ "python": ("https://docs.python.org/3", None),
+}
diff --git a/doc/source/index.rst b/doc/source/index.rst
new file mode 100644
index 00000000..e3bce61d
--- /dev/null
+++ b/doc/source/index.rst
@@ -0,0 +1,13 @@
+.. include:: ../../README.rst
+
+.. toctree::
+ :maxdepth: 2
+
+ api
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..44992853
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "readability-lxml"
+version = "0.8.4.1"
+description = "fast html to text parser (article readability tool) with python 3 support"
+authors = ["Yuri Baburov "]
+license = "Apache License 2.0"
+readme = "README.md"
+packages = [
+ { include = "readability" },
+]
+
+[tool.poetry.dependencies]
+python = ">=3.8.2,<3.14"
+chardet = "^5.2.0"
+cssselect = [
+ { version = "~1.2", markers = "python_version < '3.9'" },
+ { version = "~1.3", markers = "python_version >= '3.9'" }
+]
+lxml = {extras = ["html-clean"], version = "^5.4.0"}
+lxml-html-clean = {markers = "python_version < \"3.11\"", version = "^0.4.2"}
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/readability/BeautifulSoup.py b/readability/BeautifulSoup.py
deleted file mode 100644
index 34204e74..00000000
--- a/readability/BeautifulSoup.py
+++ /dev/null
@@ -1,2000 +0,0 @@
-"""Beautiful Soup
-Elixir and Tonic
-"The Screen-Scraper's Friend"
-http://www.crummy.com/software/BeautifulSoup/
-
-Beautiful Soup parses a (possibly invalid) XML or HTML document into a
-tree representation. It provides methods and Pythonic idioms that make
-it easy to navigate, search, and modify the tree.
-
-A well-formed XML/HTML document yields a well-formed data
-structure. An ill-formed XML/HTML document yields a correspondingly
-ill-formed data structure. If your document is only locally
-well-formed, you can use this library to find and process the
-well-formed part of it.
-
-Beautiful Soup works with Python 2.2 and up. It has no external
-dependencies, but you'll have more success at converting data to UTF-8
-if you also install these three packages:
-
-* chardet, for auto-detecting character encodings
- http://chardet.feedparser.org/
-* cjkcodecs and iconv_codec, which add more encodings to the ones supported
- by stock Python.
- http://cjkpython.i18n.org/
-
-Beautiful Soup defines classes for two main parsing strategies:
-
- * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
- language that kind of looks like XML.
-
- * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
- or invalid. This class has web browser-like heuristics for
- obtaining a sensible parse tree in the face of common HTML errors.
-
-Beautiful Soup also defines a class (UnicodeDammit) for autodetecting
-the encoding of an HTML or XML document, and converting it to
-Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser.
-
-For more than you ever wanted to know about Beautiful Soup, see the
-documentation:
-http://www.crummy.com/software/BeautifulSoup/documentation.html
-
-Here, have some legalese:
-
-Copyright (c) 2004-2009, Leonard Richardson
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- * Neither the name of the the Beautiful Soup Consortium and All
- Night Kosher Bakery nor the names of its contributors may be
- used to endorse or promote products derived from this software
- without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT.
-
-"""
-from __future__ import generators
-
-__author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "3.1.0.1"
-__copyright__ = "Copyright (c) 2004-2009 Leonard Richardson"
-__license__ = "New-style BSD"
-
-import codecs
-import markupbase
-import types
-import re
-from HTMLParser import HTMLParser, HTMLParseError
-try:
- from htmlentitydefs import name2codepoint
-except ImportError:
- name2codepoint = {}
-try:
- set
-except NameError:
- from sets import Set as set
-
-#These hacks make Beautiful Soup able to parse XML with namespaces
-markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match
-
-DEFAULT_OUTPUT_ENCODING = "utf-8"
-
-# First, the classes that represent markup elements.
-
-def sob(unicode, encoding):
- """Returns either the given Unicode string or its encoding."""
- if encoding is None:
- return unicode
- else:
- return unicode.encode(encoding)
-
-class PageElement:
- """Contains the navigational information for some part of the page
- (either a tag or a piece of text)"""
-
- def setup(self, parent=None, previous=None):
- """Sets up the initial relations between this element and
- other elements."""
- self.parent = parent
- self.previous = previous
- self.next = None
- self.previousSibling = None
- self.nextSibling = None
- if self.parent and self.parent.contents:
- self.previousSibling = self.parent.contents[-1]
- self.previousSibling.nextSibling = self
-
- def replaceWith(self, replaceWith):
- oldParent = self.parent
- myIndex = self.parent.contents.index(self)
- if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent:
- # We're replacing this element with one of its siblings.
- index = self.parent.contents.index(replaceWith)
- if index and index < myIndex:
- # Furthermore, it comes before this element. That
- # means that when we extract it, the index of this
- # element will change.
- myIndex = myIndex - 1
- self.extract()
- oldParent.insert(myIndex, replaceWith)
-
- def extract(self):
- """Destructively rips this element out of the tree."""
- if self.parent:
- try:
- self.parent.contents.remove(self)
- except ValueError:
- pass
-
- #Find the two elements that would be next to each other if
- #this element (and any children) hadn't been parsed. Connect
- #the two.
- lastChild = self._lastRecursiveChild()
- nextElement = lastChild.next
-
- if self.previous:
- self.previous.next = nextElement
- if nextElement:
- nextElement.previous = self.previous
- self.previous = None
- lastChild.next = None
-
- self.parent = None
- if self.previousSibling:
- self.previousSibling.nextSibling = self.nextSibling
- if self.nextSibling:
- self.nextSibling.previousSibling = self.previousSibling
- self.previousSibling = self.nextSibling = None
- return self
-
- def _lastRecursiveChild(self):
- "Finds the last element beneath this object to be parsed."
- lastChild = self
- while hasattr(lastChild, 'contents') and lastChild.contents:
- lastChild = lastChild.contents[-1]
- return lastChild
-
- def insert(self, position, newChild):
- if (isinstance(newChild, basestring)
- or isinstance(newChild, unicode)) \
- and not isinstance(newChild, NavigableString):
- newChild = NavigableString(newChild)
-
- position = min(position, len(self.contents))
- if hasattr(newChild, 'parent') and newChild.parent != None:
- # We're 'inserting' an element that's already one
- # of this object's children.
- if newChild.parent == self:
- index = self.find(newChild)
- if index and index < position:
- # Furthermore we're moving it further down the
- # list of this object's children. That means that
- # when we extract this element, our target index
- # will jump down one.
- position = position - 1
- newChild.extract()
-
- newChild.parent = self
- previousChild = None
- if position == 0:
- newChild.previousSibling = None
- newChild.previous = self
- else:
- previousChild = self.contents[position-1]
- newChild.previousSibling = previousChild
- newChild.previousSibling.nextSibling = newChild
- newChild.previous = previousChild._lastRecursiveChild()
- if newChild.previous:
- newChild.previous.next = newChild
-
- newChildsLastElement = newChild._lastRecursiveChild()
-
- if position >= len(self.contents):
- newChild.nextSibling = None
-
- parent = self
- parentsNextSibling = None
- while not parentsNextSibling:
- parentsNextSibling = parent.nextSibling
- parent = parent.parent
- if not parent: # This is the last element in the document.
- break
- if parentsNextSibling:
- newChildsLastElement.next = parentsNextSibling
- else:
- newChildsLastElement.next = None
- else:
- nextChild = self.contents[position]
- newChild.nextSibling = nextChild
- if newChild.nextSibling:
- newChild.nextSibling.previousSibling = newChild
- newChildsLastElement.next = nextChild
-
- if newChildsLastElement.next:
- newChildsLastElement.next.previous = newChildsLastElement
- self.contents.insert(position, newChild)
-
- def append(self, tag):
- """Appends the given tag to the contents of this tag."""
- self.insert(len(self.contents), tag)
-
- def findNext(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the first item that matches the given criteria and
- appears after this Tag in the document."""
- return self._findOne(self.findAllNext, name, attrs, text, **kwargs)
-
- def findAllNext(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns all items that match the given criteria and appear
- after this Tag in the document."""
- return self._findAll(name, attrs, text, limit, self.nextGenerator,
- **kwargs)
-
- def findNextSibling(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the closest sibling to this Tag that matches the
- given criteria and appears after this Tag in the document."""
- return self._findOne(self.findNextSiblings, name, attrs, text,
- **kwargs)
-
- def findNextSiblings(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns the siblings of this Tag that match the given
- criteria and appear after this Tag in the document."""
- return self._findAll(name, attrs, text, limit,
- self.nextSiblingGenerator, **kwargs)
- fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x
-
- def findPrevious(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the first item that matches the given criteria and
- appears before this Tag in the document."""
- return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs)
-
- def findAllPrevious(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns all items that match the given criteria and appear
- before this Tag in the document."""
- return self._findAll(name, attrs, text, limit, self.previousGenerator,
- **kwargs)
- fetchPrevious = findAllPrevious # Compatibility with pre-3.x
-
- def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the closest sibling to this Tag that matches the
- given criteria and appears before this Tag in the document."""
- return self._findOne(self.findPreviousSiblings, name, attrs, text,
- **kwargs)
-
- def findPreviousSiblings(self, name=None, attrs={}, text=None,
- limit=None, **kwargs):
- """Returns the siblings of this Tag that match the given
- criteria and appear before this Tag in the document."""
- return self._findAll(name, attrs, text, limit,
- self.previousSiblingGenerator, **kwargs)
- fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x
-
- def findParent(self, name=None, attrs={}, **kwargs):
- """Returns the closest parent of this Tag that matches the given
- criteria."""
- # NOTE: We can't use _findOne because findParents takes a different
- # set of arguments.
- r = None
- l = self.findParents(name, attrs, 1)
- if l:
- r = l[0]
- return r
-
- def findParents(self, name=None, attrs={}, limit=None, **kwargs):
- """Returns the parents of this Tag that match the given
- criteria."""
-
- return self._findAll(name, attrs, None, limit, self.parentGenerator,
- **kwargs)
- fetchParents = findParents # Compatibility with pre-3.x
-
- #These methods do the real heavy lifting.
-
- def _findOne(self, method, name, attrs, text, **kwargs):
- r = None
- l = method(name, attrs, text, 1, **kwargs)
- if l:
- r = l[0]
- return r
-
- def _findAll(self, name, attrs, text, limit, generator, **kwargs):
- "Iterates over a generator looking for things that match."
-
- if isinstance(name, SoupStrainer):
- strainer = name
- else:
- # Build a SoupStrainer
- strainer = SoupStrainer(name, attrs, text, **kwargs)
- results = ResultSet(strainer)
- g = generator()
- while True:
- try:
- i = g.next()
- except StopIteration:
- break
- if i:
- found = strainer.search(i)
- if found:
- results.append(found)
- if limit and len(results) >= limit:
- break
- return results
-
- #These Generators can be used to navigate starting from both
- #NavigableStrings and Tags.
- def nextGenerator(self):
- i = self
- while i:
- i = i.next
- yield i
-
- def nextSiblingGenerator(self):
- i = self
- while i:
- i = i.nextSibling
- yield i
-
- def previousGenerator(self):
- i = self
- while i:
- i = i.previous
- yield i
-
- def previousSiblingGenerator(self):
- i = self
- while i:
- i = i.previousSibling
- yield i
-
- def parentGenerator(self):
- i = self
- while i:
- i = i.parent
- yield i
-
- # Utility methods
- def substituteEncoding(self, str, encoding=None):
- encoding = encoding or "utf-8"
- return str.replace("%SOUP-ENCODING%", encoding)
-
- def toEncoding(self, s, encoding=None):
- """Encodes an object to a string in some encoding, or to Unicode.
- ."""
- if isinstance(s, unicode):
- if encoding:
- s = s.encode(encoding)
- elif isinstance(s, str):
- if encoding:
- s = s.encode(encoding)
- else:
- s = unicode(s)
- else:
- if encoding:
- s = self.toEncoding(str(s), encoding)
- else:
- s = unicode(s)
- return s
-
-class NavigableString(unicode, PageElement):
-
- def __new__(cls, value):
- """Create a new NavigableString.
-
- When unpickling a NavigableString, this method is called with
- the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be
- passed in to the superclass's __new__ or the superclass won't know
- how to handle non-ASCII characters.
- """
- if isinstance(value, unicode):
- return unicode.__new__(cls, value)
- return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
-
- def __getnewargs__(self):
- return (unicode(self),)
-
- def __getattr__(self, attr):
- """text.string gives you text. This is for backwards
- compatibility for Navigable*String, but for CData* it lets you
- get the string without the CData wrapper."""
- if attr == 'string':
- return self
- else:
- raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
-
- def encode(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return self.decode().encode(encoding)
-
- def decodeGivenEventualEncoding(self, eventualEncoding):
- return self
-
-class CData(NavigableString):
-
- def decodeGivenEventualEncoding(self, eventualEncoding):
- return u''
-
-class ProcessingInstruction(NavigableString):
-
- def decodeGivenEventualEncoding(self, eventualEncoding):
- output = self
- if u'%SOUP-ENCODING%' in output:
- output = self.substituteEncoding(output, eventualEncoding)
- return u'' + output + u'?>'
-
-class Comment(NavigableString):
- def decodeGivenEventualEncoding(self, eventualEncoding):
- return u''
-
-class Declaration(NavigableString):
- def decodeGivenEventualEncoding(self, eventualEncoding):
- return u''
-
-class Tag(PageElement):
-
- """Represents a found HTML tag with its attributes and contents."""
-
- def _invert(h):
- "Cheap function to invert a hash."
- i = {}
- for k,v in h.items():
- i[v] = k
- return i
-
- XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'",
- "quot" : '"',
- "amp" : "&",
- "lt" : "<",
- "gt" : ">" }
-
- XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS)
-
- def _convertEntities(self, match):
- """Used in a call to re.sub to replace HTML, XML, and numeric
- entities with the appropriate Unicode characters. If HTML
- entities are being converted, any unrecognized entities are
- escaped."""
- x = match.group(1)
- if self.convertHTMLEntities and x in name2codepoint:
- return unichr(name2codepoint[x])
- elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS:
- if self.convertXMLEntities:
- return self.XML_ENTITIES_TO_SPECIAL_CHARS[x]
- else:
- return u'&%s;' % x
- elif len(x) > 0 and x[0] == '#':
- # Handle numeric entities
- if len(x) > 1 and x[1] == 'x':
- return unichr(int(x[2:], 16))
- else:
- return unichr(int(x[1:]))
-
- elif self.escapeUnrecognizedEntities:
- return u'&%s;' % x
- else:
- return u'&%s;' % x
-
- def __init__(self, parser, name, attrs=None, parent=None,
- previous=None):
- "Basic constructor."
-
- # We don't actually store the parser object: that lets extracted
- # chunks be garbage-collected
- self.parserClass = parser.__class__
- self.isSelfClosing = parser.isSelfClosingTag(name)
- self.name = name
- if attrs == None:
- attrs = []
- self.attrs = attrs
- self.contents = []
- self.setup(parent, previous)
- self.hidden = False
- self.containsSubstitutions = False
- self.convertHTMLEntities = parser.convertHTMLEntities
- self.convertXMLEntities = parser.convertXMLEntities
- self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities
-
- def convert(kval):
- "Converts HTML, XML and numeric entities in the attribute value."
- k, val = kval
- if val is None:
- return kval
- return (k, re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);",
- self._convertEntities, val))
- self.attrs = map(convert, self.attrs)
-
- def get(self, key, default=None):
- """Returns the value of the 'key' attribute for the tag, or
- the value given for 'default' if it doesn't have that
- attribute."""
- return self._getAttrMap().get(key, default)
-
- def has_key(self, key):
- return self._getAttrMap().has_key(key)
-
- def __getitem__(self, key):
- """tag[key] returns the value of the 'key' attribute for the tag,
- and throws an exception if it's not there."""
- return self._getAttrMap()[key]
-
- def __iter__(self):
- "Iterating over a tag iterates over its contents."
- return iter(self.contents)
-
- def __len__(self):
- "The length of a tag is the length of its list of contents."
- return len(self.contents)
-
- def __contains__(self, x):
- return x in self.contents
-
- def __nonzero__(self):
- "A tag is non-None even if it has no contents."
- return True
-
- def __setitem__(self, key, value):
- """Setting tag[key] sets the value of the 'key' attribute for the
- tag."""
- self._getAttrMap()
- self.attrMap[key] = value
- found = False
- for i in range(0, len(self.attrs)):
- if self.attrs[i][0] == key:
- self.attrs[i] = (key, value)
- found = True
- if not found:
- self.attrs.append((key, value))
- self._getAttrMap()[key] = value
-
- def __delitem__(self, key):
- "Deleting tag[key] deletes all 'key' attributes for the tag."
- for item in self.attrs:
- if item[0] == key:
- self.attrs.remove(item)
- #We don't break because bad HTML can define the same
- #attribute multiple times.
- self._getAttrMap()
- if self.attrMap.has_key(key):
- del self.attrMap[key]
-
- def __call__(self, *args, **kwargs):
- """Calling a tag like a function is the same as calling its
- findAll() method. Eg. tag('a') returns a list of all the A tags
- found within this tag."""
- return apply(self.findAll, args, kwargs)
-
- def __getattr__(self, tag):
- #print "Getattr %s.%s" % (self.__class__, tag)
- if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3:
- return self.find(tag[:-3])
- elif tag.find('__') != 0:
- return self.find(tag)
- raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag)
-
- def __eq__(self, other):
- """Returns true iff this tag has the same name, the same attributes,
- and the same contents (recursively) as the given tag.
-
- NOTE: right now this will return false if two tags have the
- same attributes in a different order. Should this be fixed?"""
- if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
- return False
- for i in range(0, len(self.contents)):
- if self.contents[i] != other.contents[i]:
- return False
- return True
-
- def __ne__(self, other):
- """Returns true iff this tag is not identical to the other tag,
- as defined in __eq__."""
- return not self == other
-
- def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- """Renders this tag as a string."""
- return self.decode(eventualEncoding=encoding)
-
- BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
- + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
- + ")")
-
- def _sub_entity(self, x):
- """Used with a regular expression to substitute the
- appropriate XML entity for an XML special character."""
- return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";"
-
- def __unicode__(self):
- return self.decode()
-
- def __str__(self):
- return self.encode()
-
- def encode(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- return self.decode(prettyPrint, indentLevel, encoding).encode(encoding)
-
- def decode(self, prettyPrint=False, indentLevel=0,
- eventualEncoding=DEFAULT_OUTPUT_ENCODING):
- """Returns a string or Unicode representation of this tag and
- its contents. To get Unicode, pass None for encoding."""
-
- attrs = []
- if self.attrs:
- for key, val in self.attrs:
- fmt = '%s="%s"'
- if isString(val):
- if (self.containsSubstitutions
- and eventualEncoding is not None
- and '%SOUP-ENCODING%' in val):
- val = self.substituteEncoding(val, eventualEncoding)
-
- # The attribute value either:
- #
- # * Contains no embedded double quotes or single quotes.
- # No problem: we enclose it in double quotes.
- # * Contains embedded single quotes. No problem:
- # double quotes work here too.
- # * Contains embedded double quotes. No problem:
- # we enclose it in single quotes.
- # * Embeds both single _and_ double quotes. This
- # can't happen naturally, but it can happen if
- # you modify an attribute value after parsing
- # the document. Now we have a bit of a
- # problem. We solve it by enclosing the
- # attribute in single quotes, and escaping any
- # embedded single quotes to XML entities.
- if '"' in val:
- fmt = "%s='%s'"
- if "'" in val:
- # TODO: replace with apos when
- # appropriate.
- val = val.replace("'", "&squot;")
-
- # Now we're okay w/r/t quotes. But the attribute
- # value might also contain angle brackets, or
- # ampersands that aren't part of entities. We need
- # to escape those to XML entities too.
- val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val)
- if val is None:
- # Handle boolean attributes.
- decoded = key
- else:
- decoded = fmt % (key, val)
- attrs.append(decoded)
- close = ''
- closeTag = ''
- if self.isSelfClosing:
- close = ' /'
- else:
- closeTag = '%s>' % self.name
-
- indentTag, indentContents = 0, 0
- if prettyPrint:
- indentTag = indentLevel
- space = (' ' * (indentTag-1))
- indentContents = indentTag + 1
- contents = self.decodeContents(prettyPrint, indentContents,
- eventualEncoding)
- if self.hidden:
- s = contents
- else:
- s = []
- attributeString = ''
- if attrs:
- attributeString = ' ' + ' '.join(attrs)
- if prettyPrint:
- s.append(space)
- s.append('<%s%s%s>' % (self.name, attributeString, close))
- if prettyPrint:
- s.append("\n")
- s.append(contents)
- if prettyPrint and contents and contents[-1] != "\n":
- s.append("\n")
- if prettyPrint and closeTag:
- s.append(space)
- s.append(closeTag)
- if prettyPrint and closeTag and self.nextSibling:
- s.append("\n")
- s = ''.join(s)
- return s
-
- def decompose(self):
- """Recursively destroys the contents of this tree."""
- contents = [i for i in self.contents]
- for i in contents:
- if isinstance(i, Tag):
- i.decompose()
- else:
- i.extract()
- self.extract()
-
- def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return self.encode(encoding, True)
-
- def encodeContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- return self.decodeContents(prettyPrint, indentLevel).encode(encoding)
-
- def decodeContents(self, prettyPrint=False, indentLevel=0,
- eventualEncoding=DEFAULT_OUTPUT_ENCODING):
- """Renders the contents of this tag as a string in the given
- encoding. If encoding is None, returns a Unicode string.."""
- s=[]
- for c in self:
- text = None
- if isinstance(c, NavigableString):
- text = c.decodeGivenEventualEncoding(eventualEncoding)
- elif isinstance(c, Tag):
- s.append(c.decode(prettyPrint, indentLevel, eventualEncoding))
- if text and prettyPrint:
- text = text.strip()
- if text:
- if prettyPrint:
- s.append(" " * (indentLevel-1))
- s.append(text)
- if prettyPrint:
- s.append("\n")
- return ''.join(s)
-
- #Soup methods
-
- def find(self, name=None, attrs={}, recursive=True, text=None,
- **kwargs):
- """Return only the first child of this Tag matching the given
- criteria."""
- r = None
- l = self.findAll(name, attrs, recursive, text, 1, **kwargs)
- if l:
- r = l[0]
- return r
- findChild = find
-
- def findAll(self, name=None, attrs={}, recursive=True, text=None,
- limit=None, **kwargs):
- """Extracts a list of Tag objects that match the given
- criteria. You can specify the name of the Tag and any
- attributes you want the Tag to have.
-
- The value of a key-value pair in the 'attrs' map can be a
- string, a list of strings, a regular expression object, or a
- callable that takes a string and returns whether or not the
- string matches for some custom definition of 'matches'. The
- same is true of the tag name."""
- generator = self.recursiveChildGenerator
- if not recursive:
- generator = self.childGenerator
- return self._findAll(name, attrs, text, limit, generator, **kwargs)
- findChildren = findAll
-
- # Pre-3.x compatibility methods. Will go away in 4.0.
- first = find
- fetch = findAll
-
- def fetchText(self, text=None, recursive=True, limit=None):
- return self.findAll(text=text, recursive=recursive, limit=limit)
-
- def firstText(self, text=None, recursive=True):
- return self.find(text=text, recursive=recursive)
-
- # 3.x compatibility methods. Will go away in 4.0.
- def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- if encoding is None:
- return self.decodeContents(prettyPrint, indentLevel, encoding)
- else:
- return self.encodeContents(encoding, prettyPrint, indentLevel)
-
-
- #Private methods
-
- def _getAttrMap(self):
- """Initializes a map representation of this tag's attributes,
- if not already initialized."""
- if not getattr(self, 'attrMap'):
- self.attrMap = {}
- for (key, value) in self.attrs:
- self.attrMap[key] = value
- return self.attrMap
-
- #Generator methods
- def recursiveChildGenerator(self):
- if not len(self.contents):
- raise StopIteration
- stopNode = self._lastRecursiveChild().next
- current = self.contents[0]
- while current is not stopNode:
- yield current
- current = current.next
-
- def childGenerator(self):
- if not len(self.contents):
- raise StopIteration
- current = self.contents[0]
- while current:
- yield current
- current = current.nextSibling
- raise StopIteration
-
-# Next, a couple classes to represent queries and their results.
-class SoupStrainer:
- """Encapsulates a number of ways of matching a markup element (tag or
- text)."""
-
- def __init__(self, name=None, attrs={}, text=None, **kwargs):
- self.name = name
- if isString(attrs):
- kwargs['class'] = attrs
- attrs = None
- if kwargs:
- if attrs:
- attrs = attrs.copy()
- attrs.update(kwargs)
- else:
- attrs = kwargs
- self.attrs = attrs
- self.text = text
-
- def __str__(self):
- if self.text:
- return self.text
- else:
- return "%s|%s" % (self.name, self.attrs)
-
- def searchTag(self, markupName=None, markupAttrs={}):
- found = None
- markup = None
- if isinstance(markupName, Tag):
- markup = markupName
- markupAttrs = markup
- callFunctionWithTagData = callable(self.name) \
- and not isinstance(markupName, Tag)
-
- if (not self.name) \
- or callFunctionWithTagData \
- or (markup and self._matches(markup, self.name)) \
- or (not markup and self._matches(markupName, self.name)):
- if callFunctionWithTagData:
- match = self.name(markupName, markupAttrs)
- else:
- match = True
- markupAttrMap = None
- for attr, matchAgainst in self.attrs.items():
- if not markupAttrMap:
- if hasattr(markupAttrs, 'get'):
- markupAttrMap = markupAttrs
- else:
- markupAttrMap = {}
- for k,v in markupAttrs:
- markupAttrMap[k] = v
- attrValue = markupAttrMap.get(attr)
- if not self._matches(attrValue, matchAgainst):
- match = False
- break
- if match:
- if markup:
- found = markup
- else:
- found = markupName
- return found
-
- def search(self, markup):
- #print 'looking for %s in %s' % (self, markup)
- found = None
- # If given a list of items, scan it for a text element that
- # matches.
- if isList(markup) and not isinstance(markup, Tag):
- for element in markup:
- if isinstance(element, NavigableString) \
- and self.search(element):
- found = element
- break
- # If it's a Tag, make sure its name or attributes match.
- # Don't bother with Tags if we're searching for text.
- elif isinstance(markup, Tag):
- if not self.text:
- found = self.searchTag(markup)
- # If it's text, make sure the text matches.
- elif isinstance(markup, NavigableString) or \
- isString(markup):
- if self._matches(markup, self.text):
- found = markup
- else:
- raise Exception, "I don't know how to match against a %s" \
- % markup.__class__
- return found
-
- def _matches(self, markup, matchAgainst):
- #print "Matching %s against %s" % (markup, matchAgainst)
- result = False
- if matchAgainst == True and type(matchAgainst) == types.BooleanType:
- result = markup != None
- elif callable(matchAgainst):
- result = matchAgainst(markup)
- else:
- #Custom match methods take the tag as an argument, but all
- #other ways of matching match the tag name as a string.
- if isinstance(markup, Tag):
- markup = markup.name
- if markup is not None and not isString(markup):
- markup = unicode(markup)
- #Now we know that chunk is either a string, or None.
- if hasattr(matchAgainst, 'match'):
- # It's a regexp object.
- result = markup and matchAgainst.search(markup)
- elif (isList(matchAgainst)
- and (markup is not None or not isString(matchAgainst))):
- result = markup in matchAgainst
- elif hasattr(matchAgainst, 'items'):
- result = markup.has_key(matchAgainst)
- elif matchAgainst and isString(markup):
- if isinstance(markup, unicode):
- matchAgainst = unicode(matchAgainst)
- else:
- matchAgainst = str(matchAgainst)
-
- if not result:
- result = matchAgainst == markup
- return result
-
-class ResultSet(list):
- """A ResultSet is just a list that keeps track of the SoupStrainer
- that created it."""
- def __init__(self, source):
- list.__init__([])
- self.source = source
-
-# Now, some helper functions.
-
-def isList(l):
- """Convenience method that works with all 2.x versions of Python
- to determine whether or not something is listlike."""
- return ((hasattr(l, '__iter__') and not isString(l))
- or (type(l) in (types.ListType, types.TupleType)))
-
-def isString(s):
- """Convenience method that works with all 2.x versions of Python
- to determine whether or not something is stringlike."""
- try:
- return isinstance(s, unicode) or isinstance(s, basestring)
- except NameError:
- return isinstance(s, str)
-
-def buildTagMap(default, *args):
- """Turns a list of maps, lists, or scalars into a single map.
- Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and
- NESTING_RESET_TAGS maps out of lists and partial maps."""
- built = {}
- for portion in args:
- if hasattr(portion, 'items'):
- #It's a map. Merge it.
- for k,v in portion.items():
- built[k] = v
- elif isList(portion) and not isString(portion):
- #It's a list. Map each item to the default.
- for k in portion:
- built[k] = default
- else:
- #It's a scalar. Map it to the default.
- built[portion] = default
- return built
-
-# Now, the parser classes.
-
-class HTMLParserBuilder(HTMLParser):
-
- def __init__(self, soup):
- HTMLParser.__init__(self)
- self.soup = soup
-
- # We inherit feed() and reset().
-
- def handle_starttag(self, name, attrs):
- if name == 'meta':
- self.soup.extractCharsetFromMeta(attrs)
- else:
- self.soup.unknown_starttag(name, attrs)
-
- def handle_endtag(self, name):
- self.soup.unknown_endtag(name)
-
- def handle_data(self, content):
- self.soup.handle_data(content)
-
- def _toStringSubclass(self, text, subclass):
- """Adds a certain piece of text to the tree as a NavigableString
- subclass."""
- self.soup.endData()
- self.handle_data(text)
- self.soup.endData(subclass)
-
- def handle_pi(self, text):
- """Handle a processing instruction as a ProcessingInstruction
- object, possibly one with a %SOUP-ENCODING% slot into which an
- encoding will be plugged later."""
- if text[:3] == "xml":
- text = u"xml version='1.0' encoding='%SOUP-ENCODING%'"
- self._toStringSubclass(text, ProcessingInstruction)
-
- def handle_comment(self, text):
- "Handle comments as Comment objects."
- self._toStringSubclass(text, Comment)
-
- def handle_charref(self, ref):
- "Handle character references as data."
- if self.soup.convertEntities:
- data = unichr(int(ref))
- else:
- data = '%s;' % ref
- self.handle_data(data)
-
- def handle_entityref(self, ref):
- """Handle entity references as data, possibly converting known
- HTML and/or XML entity references to the corresponding Unicode
- characters."""
- data = None
- if self.soup.convertHTMLEntities:
- try:
- data = unichr(name2codepoint[ref])
- except KeyError:
- pass
-
- if not data and self.soup.convertXMLEntities:
- data = self.soup.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref)
-
- if not data and self.soup.convertHTMLEntities and \
- not self.soup.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref):
- # TODO: We've got a problem here. We're told this is
- # an entity reference, but it's not an XML entity
- # reference or an HTML entity reference. Nonetheless,
- # the logical thing to do is to pass it through as an
- # unrecognized entity reference.
- #
- # Except: when the input is "&carol;" this function
- # will be called with input "carol". When the input is
- # "AT&T", this function will be called with input
- # "T". We have no way of knowing whether a semicolon
- # was present originally, so we don't know whether
- # this is an unknown entity or just a misplaced
- # ampersand.
- #
- # The more common case is a misplaced ampersand, so I
- # escape the ampersand and omit the trailing semicolon.
- data = "&%s" % ref
- if not data:
- # This case is different from the one above, because we
- # haven't already gone through a supposedly comprehensive
- # mapping of entities to Unicode characters. We might not
- # have gone through any mapping at all. So the chances are
- # very high that this is a real entity, and not a
- # misplaced ampersand.
- data = "&%s;" % ref
- self.handle_data(data)
-
- def handle_decl(self, data):
- "Handle DOCTYPEs and the like as Declaration objects."
- self._toStringSubclass(data, Declaration)
-
- def parse_declaration(self, i):
- """Treat a bogus SGML declaration as raw data. Treat a CDATA
- declaration as a CData object."""
- j = None
- if self.rawdata[i:i+9] == '', i)
- if k == -1:
- k = len(self.rawdata)
- data = self.rawdata[i+9:k]
- j = k+3
- self._toStringSubclass(data, CData)
- else:
- try:
- j = HTMLParser.parse_declaration(self, i)
- except HTMLParseError:
- toHandle = self.rawdata[i:]
- self.handle_data(toHandle)
- j = i + len(toHandle)
- return j
-
-
-class BeautifulStoneSoup(Tag):
-
- """This class contains the basic parser and search code. It defines
- a parser that knows nothing about tag behavior except for the
- following:
-
- You can't close a tag without closing all the tags it encloses.
- That is, "" actually means
- "".
-
- [Another possible explanation is "", but since
- this class defines no SELF_CLOSING_TAGS, it will never use that
- explanation.]
-
- This class is useful for parsing XML or made-up markup languages,
- or when BeautifulSoup makes an assumption counter to what you were
- expecting."""
-
- SELF_CLOSING_TAGS = {}
- NESTABLE_TAGS = {}
- RESET_NESTING_TAGS = {}
- QUOTE_TAGS = {}
- PRESERVE_WHITESPACE_TAGS = []
-
- MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'),
- lambda x: x.group(1) + ' />'),
- (re.compile(']*)>'),
- lambda x: '')
- ]
-
- ROOT_TAG_NAME = u'[document]'
-
- HTML_ENTITIES = "html"
- XML_ENTITIES = "xml"
- XHTML_ENTITIES = "xhtml"
- # TODO: This only exists for backwards-compatibility
- ALL_ENTITIES = XHTML_ENTITIES
-
- # Used when determining whether a text node is all whitespace and
- # can be replaced with a single space. A text node that contains
- # fancy Unicode spaces (usually non-breaking) should be left
- # alone.
- STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, }
-
- def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None,
- markupMassage=True, smartQuotesTo=XML_ENTITIES,
- convertEntities=None, selfClosingTags=None, isHTML=False,
- builder=HTMLParserBuilder):
- """The Soup object is initialized as the 'root tag', and the
- provided markup (which can be a string or a file-like object)
- is fed into the underlying parser.
-
- HTMLParser will process most bad HTML, and the BeautifulSoup
- class has some tricks for dealing with some HTML that kills
- HTMLParser, but Beautiful Soup can nonetheless choke or lose data
- if your data uses self-closing tags or declarations
- incorrectly.
-
- By default, Beautiful Soup uses regexes to sanitize input,
- avoiding the vast majority of these problems. If the problems
- don't apply to you, pass in False for markupMassage, and
- you'll get better performance.
-
- The default parser massage techniques fix the two most common
- instances of invalid HTML that choke HTMLParser:
-
- (No space between name of closing tag and tag close)
- (Extraneous whitespace in declaration)
-
- You can pass in a custom list of (RE object, replace method)
- tuples to get Beautiful Soup to scrub your input the way you
- want."""
-
- self.parseOnlyThese = parseOnlyThese
- self.fromEncoding = fromEncoding
- self.smartQuotesTo = smartQuotesTo
- self.convertEntities = convertEntities
- # Set the rules for how we'll deal with the entities we
- # encounter
- if self.convertEntities:
- # It doesn't make sense to convert encoded characters to
- # entities even while you're converting entities to Unicode.
- # Just convert it all to Unicode.
- self.smartQuotesTo = None
- if convertEntities == self.HTML_ENTITIES:
- self.convertXMLEntities = False
- self.convertHTMLEntities = True
- self.escapeUnrecognizedEntities = True
- elif convertEntities == self.XHTML_ENTITIES:
- self.convertXMLEntities = True
- self.convertHTMLEntities = True
- self.escapeUnrecognizedEntities = False
- elif convertEntities == self.XML_ENTITIES:
- self.convertXMLEntities = True
- self.convertHTMLEntities = False
- self.escapeUnrecognizedEntities = False
- else:
- self.convertXMLEntities = False
- self.convertHTMLEntities = False
- self.escapeUnrecognizedEntities = False
-
- self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags)
- self.builder = builder(self)
- self.reset()
-
- if hasattr(markup, 'read'): # It's a file-type object.
- markup = markup.read()
- self.markup = markup
- self.markupMassage = markupMassage
- try:
- self._feed(isHTML=isHTML)
- except StopParsing:
- pass
- self.markup = None # The markup can now be GCed.
- self.builder = None # So can the builder.
-
- def _feed(self, inDocumentEncoding=None, isHTML=False):
- # Convert the document to Unicode.
- markup = self.markup
- if isinstance(markup, unicode):
- if not hasattr(self, 'originalEncoding'):
- self.originalEncoding = None
- else:
- dammit = UnicodeDammit\
- (markup, [self.fromEncoding, inDocumentEncoding],
- smartQuotesTo=self.smartQuotesTo, isHTML=isHTML)
- markup = dammit.unicode
- self.originalEncoding = dammit.originalEncoding
- self.declaredHTMLEncoding = dammit.declaredHTMLEncoding
- if markup:
- if self.markupMassage:
- if not isList(self.markupMassage):
- self.markupMassage = self.MARKUP_MASSAGE
- for fix, m in self.markupMassage:
- markup = fix.sub(m, markup)
- # TODO: We get rid of markupMassage so that the
- # soup object can be deepcopied later on. Some
- # Python installations can't copy regexes. If anyone
- # was relying on the existence of markupMassage, this
- # might cause problems.
- del(self.markupMassage)
- self.builder.reset()
-
- self.builder.feed(markup)
- # Close out any unfinished strings and close all the open tags.
- self.endData()
- while self.currentTag.name != self.ROOT_TAG_NAME:
- self.popTag()
-
- def isSelfClosingTag(self, name):
- """Returns true iff the given string is the name of a
- self-closing tag according to this parser."""
- return self.SELF_CLOSING_TAGS.has_key(name) \
- or self.instanceSelfClosingTags.has_key(name)
-
- def reset(self):
- Tag.__init__(self, self, self.ROOT_TAG_NAME)
- self.hidden = 1
- self.builder.reset()
- self.currentData = []
- self.currentTag = None
- self.tagStack = []
- self.quoteStack = []
- self.pushTag(self)
-
- def popTag(self):
- tag = self.tagStack.pop()
- # Tags with just one string-owning child get the child as a
- # 'string' property, so that soup.tag.string is shorthand for
- # soup.tag.contents[0]
- if len(self.currentTag.contents) == 1 and \
- isinstance(self.currentTag.contents[0], NavigableString):
- self.currentTag.string = self.currentTag.contents[0]
-
- #print "Pop", tag.name
- if self.tagStack:
- self.currentTag = self.tagStack[-1]
- return self.currentTag
-
- def pushTag(self, tag):
- #print "Push", tag.name
- if self.currentTag:
- self.currentTag.contents.append(tag)
- self.tagStack.append(tag)
- self.currentTag = self.tagStack[-1]
-
- def endData(self, containerClass=NavigableString):
- if self.currentData:
- currentData = u''.join(self.currentData)
- if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
- not set([tag.name for tag in self.tagStack]).intersection(
- self.PRESERVE_WHITESPACE_TAGS)):
- if '\n' in currentData:
- currentData = '\n'
- else:
- currentData = ' '
- self.currentData = []
- if self.parseOnlyThese and len(self.tagStack) <= 1 and \
- (not self.parseOnlyThese.text or \
- not self.parseOnlyThese.search(currentData)):
- return
- o = containerClass(currentData)
- o.setup(self.currentTag, self.previous)
- if self.previous:
- self.previous.next = o
- self.previous = o
- self.currentTag.contents.append(o)
-
-
- def _popToTag(self, name, inclusivePop=True):
- """Pops the tag stack up to and including the most recent
- instance of the given tag. If inclusivePop is false, pops the tag
- stack up to but *not* including the most recent instqance of
- the given tag."""
- #print "Popping to %s" % name
- if name == self.ROOT_TAG_NAME:
- return
-
- numPops = 0
- mostRecentTag = None
- for i in range(len(self.tagStack)-1, 0, -1):
- if name == self.tagStack[i].name:
- numPops = len(self.tagStack)-i
- break
- if not inclusivePop:
- numPops = numPops - 1
-
- for i in range(0, numPops):
- mostRecentTag = self.popTag()
- return mostRecentTag
-
- def _smartPop(self, name):
-
- """We need to pop up to the previous tag of this type, unless
- one of this tag's nesting reset triggers comes between this
- tag and the previous tag of this type, OR unless this tag is a
- generic nesting trigger and another generic nesting trigger
- comes between this tag and the previous tag of this type.
-
- Examples:
-
FooBar *
* should pop to 'p', not 'b'.
-
Foo
Bar *
* should pop to 'table', not 'p'.
-
Foo
Bar *
* should pop to 'tr', not 'p'.
-
-
*
* should pop to 'ul', not the first 'li'.
-
*
* should pop to 'table', not the first 'tr'
-
*
* should pop to 'tr', not the first 'td'
- """
-
- nestingResetTriggers = self.NESTABLE_TAGS.get(name)
- isNestable = nestingResetTriggers != None
- isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
- popTo = None
- inclusive = True
- for i in range(len(self.tagStack)-1, 0, -1):
- p = self.tagStack[i]
- if (not p or p.name == name) and not isNestable:
- #Non-nestable tags get popped to the top or to their
- #last occurance.
- popTo = name
- break
- if (nestingResetTriggers != None
- and p.name in nestingResetTriggers) \
- or (nestingResetTriggers == None and isResetNesting
- and self.RESET_NESTING_TAGS.has_key(p.name)):
-
- #If we encounter one of the nesting reset triggers
- #peculiar to this tag, or we encounter another tag
- #that causes nesting to reset, pop up to but not
- #including that tag.
- popTo = p.name
- inclusive = False
- break
- p = p.parent
- if popTo:
- self._popToTag(popTo, inclusive)
-
- def unknown_starttag(self, name, attrs, selfClosing=0):
- #print "Start tag %s: %s" % (name, attrs)
- if self.quoteStack:
- #This is not a real tag.
- #print "<%s> is not real!" % name
- attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs))
- self.handle_data('<%s%s>' % (name, attrs))
- return
- self.endData()
-
- if not self.isSelfClosingTag(name) and not selfClosing:
- self._smartPop(name)
-
- if self.parseOnlyThese and len(self.tagStack) <= 1 \
- and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)):
- return
-
- tag = Tag(self, name, attrs, self.currentTag, self.previous)
- if self.previous:
- self.previous.next = tag
- self.previous = tag
- self.pushTag(tag)
- if selfClosing or self.isSelfClosingTag(name):
- self.popTag()
- if name in self.QUOTE_TAGS:
- #print "Beginning quote (%s)" % name
- self.quoteStack.append(name)
- self.literal = 1
- return tag
-
- def unknown_endtag(self, name):
- #print "End tag %s" % name
- if self.quoteStack and self.quoteStack[-1] != name:
- #This is not a real end tag.
- #print "%s> is not real!" % name
- self.handle_data('%s>' % name)
- return
- self.endData()
- self._popToTag(name)
- if self.quoteStack and self.quoteStack[-1] == name:
- self.quoteStack.pop()
- self.literal = (len(self.quoteStack) > 0)
-
- def handle_data(self, data):
- self.currentData.append(data)
-
- def extractCharsetFromMeta(self, attrs):
- self.unknown_starttag('meta', attrs)
-
-
-class BeautifulSoup(BeautifulStoneSoup):
-
- """This parser knows the following facts about HTML:
-
- * Some tags have no closing tag and should be interpreted as being
- closed as soon as they are encountered.
-
- * The text inside some tags (ie. 'script') may contain tags which
- are not really part of the document and which should be parsed
- as text, not tags. If you want to parse the text as tags, you can
- always fetch it and parse it explicitly.
-
- * Tag nesting rules:
-
- Most tags can't be nested at all. For instance, the occurance of
- a
tag should implicitly close the previous
tag.
-
-
Para1
Para2
- should be transformed into:
-
Para1
Para2
-
- Some tags can be nested arbitrarily. For instance, the occurance
- of a
tag should _not_ implicitly close the previous
-
tag.
-
- Alice said:
Bob said:
Blah
- should NOT be transformed into:
- Alice said:
Bob said:
Blah
-
- Some tags can be nested, but the nesting is reset by the
- interposition of other tags. For instance, a
tag should
- implicitly close the previous
tag within the same
,
- but not close a
tag in another table.
-
-
Blah
Blah
- should be transformed into:
-
Blah
Blah
- but,
-
Blah
Blah
- should NOT be transformed into
-
Blah
Blah
-
- Differing assumptions about tag nesting rules are a major source
- of problems with the BeautifulSoup class. If BeautifulSoup is not
- treating as nestable a tag your page author treats as nestable,
- try ICantBelieveItsBeautifulSoup, MinimalSoup, or
- BeautifulStoneSoup before writing your own subclass."""
-
- def __init__(self, *args, **kwargs):
- if not kwargs.has_key('smartQuotesTo'):
- kwargs['smartQuotesTo'] = self.HTML_ENTITIES
- kwargs['isHTML'] = True
- BeautifulStoneSoup.__init__(self, *args, **kwargs)
-
- SELF_CLOSING_TAGS = buildTagMap(None,
- ['br' , 'hr', 'input', 'img', 'meta',
- 'spacer', 'link', 'frame', 'base'])
-
- PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
-
- QUOTE_TAGS = {'script' : None, 'textarea' : None}
-
- #According to the HTML standard, each of these inline tags can
- #contain another tag of the same type. Furthermore, it's common
- #to actually use these tags this way.
- NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
- 'center']
-
- #According to the HTML standard, these block tags can contain
- #another tag of the same type. Furthermore, it's common
- #to actually use these tags this way.
- NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
-
- #Lists can contain other lists, but there are restrictions.
- NESTABLE_LIST_TAGS = { 'ol' : [],
- 'ul' : [],
- 'li' : ['ul', 'ol'],
- 'dl' : [],
- 'dd' : ['dl'],
- 'dt' : ['dl'] }
-
- #Tables can contain other tables, but there are restrictions.
- NESTABLE_TABLE_TAGS = {'table' : [],
- 'tr' : ['table', 'tbody', 'tfoot', 'thead'],
- 'td' : ['tr'],
- 'th' : ['tr'],
- 'thead' : ['table'],
- 'tbody' : ['table'],
- 'tfoot' : ['table'],
- }
-
- NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre']
-
- #If one of these tags is encountered, all tags up to the next tag of
- #this type are popped.
- RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
- NON_NESTABLE_BLOCK_TAGS,
- NESTABLE_LIST_TAGS,
- NESTABLE_TABLE_TAGS)
-
- NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
- NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
-
- # Used to detect the charset in a META tag; see start_meta
- CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M)
-
- def extractCharsetFromMeta(self, attrs):
- """Beautiful Soup can detect a charset included in a META tag,
- try to convert the document to that charset, and re-parse the
- document from the beginning."""
- httpEquiv = None
- contentType = None
- contentTypeIndex = None
- tagNeedsEncodingSubstitution = False
-
- for i in range(0, len(attrs)):
- key, value = attrs[i]
- key = key.lower()
- if key == 'http-equiv':
- httpEquiv = value
- elif key == 'content':
- contentType = value
- contentTypeIndex = i
-
- if httpEquiv and contentType: # It's an interesting meta tag.
- match = self.CHARSET_RE.search(contentType)
- if match:
- if (self.declaredHTMLEncoding is not None or
- self.originalEncoding == self.fromEncoding):
- # An HTML encoding was sniffed while converting
- # the document to Unicode, or an HTML encoding was
- # sniffed during a previous pass through the
- # document, or an encoding was specified
- # explicitly and it worked. Rewrite the meta tag.
- def rewrite(match):
- return match.group(1) + "%SOUP-ENCODING%"
- newAttr = self.CHARSET_RE.sub(rewrite, contentType)
- attrs[contentTypeIndex] = (attrs[contentTypeIndex][0],
- newAttr)
- tagNeedsEncodingSubstitution = True
- else:
- # This is our first pass through the document.
- # Go through it again with the encoding information.
- newCharset = match.group(3)
- if newCharset and newCharset != self.originalEncoding:
- self.declaredHTMLEncoding = newCharset
- self._feed(self.declaredHTMLEncoding)
- raise StopParsing
- pass
- tag = self.unknown_starttag("meta", attrs)
- if tag and tagNeedsEncodingSubstitution:
- tag.containsSubstitutions = True
-
-
-class StopParsing(Exception):
- pass
-
-class ICantBelieveItsBeautifulSoup(BeautifulSoup):
-
- """The BeautifulSoup class is oriented towards skipping over
- common HTML errors like unclosed tags. However, sometimes it makes
- errors of its own. For instance, consider this fragment:
-
- FooBar
-
- This is perfectly valid (if bizarre) HTML. However, the
- BeautifulSoup class will implicitly close the first b tag when it
- encounters the second 'b'. It will think the author wrote
- "FooBar", and didn't close the first 'b' tag, because
- there's no real-world reason to bold something that's already
- bold. When it encounters '' it will close two more 'b'
- tags, for a grand total of three tags closed instead of two. This
- can throw off the rest of your document structure. The same is
- true of a number of other tags, listed below.
-
- It's much more common for someone to forget to close a 'b' tag
- than to actually use nested 'b' tags, and the BeautifulSoup class
- handles the common case. This class handles the not-co-common
- case: where you can't believe someone wrote what they did, but
- it's valid HTML and BeautifulSoup screwed up by assuming it
- wouldn't be."""
-
- I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \
- ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong',
- 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b',
- 'big']
-
- I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript']
-
- NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
- I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
- I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
-
-class MinimalSoup(BeautifulSoup):
- """The MinimalSoup class is for parsing HTML that contains
- pathologically bad markup. It makes no assumptions about tag
- nesting, but it does know which tags are self-closing, that
- ]*>', re.DOTALL | re.IGNORECASE),
- replacement=''),
-
- Replacement('double double-quoted attributes',
- regex=re.compile('(="[^"]+")"+'),
- replacement='\\1'),
-
- Replacement('unclosed tags',
- regex = re.compile('(<[a-zA-Z]+[^>]*)(<[a-zA-Z]+[^<>]*>)'),
- replacement='\\1>\\2'),
-
- Replacement('unclosed (numerical) attribute values',
- regex = re.compile('(<[^>]*[a-zA-Z]+\s*=\s*"[0-9]+)( [a-zA-Z]+="\w+"|/?>)'),
- replacement='\\1"\\2'),
- )
-
-
-# helpers for parsing
-def normalize_spaces(s):
- """replace any sequence of whitespace
- characters with a single space"""
- return ' '.join(s.split())
-
-def _remove_crufty_html(content):
- for replacement in dodgy_regexes:
- content = replacement.apply(content)
- return content
-
-def _parse_methods():
- def unicode_cleansed(content, base_href):
- content = UnicodeDammit(content, isHTML=True).markup
- cleaned = _remove_crufty_html(content)
- debug("Cleaned content: %s" % (cleaned,))
- return beautiful_soup(cleaned, base_href)
-
- def ascii_cleansed(content, base_href):
- content = ascii(content)
- cleaned = _remove_crufty_html(content)
- debug("Cleaned content: %s" % (cleaned,))
- return beautiful_soup(cleaned, base_href)
-
- return (
- beautiful_soup,
- unicode_cleansed,
- ascii_cleansed)
-
-# strip out a set of nuisance html attributes that can mess up rendering in RSS feeds
-bad_attrs = ['width','height','style','[-a-z]*color','background[-a-z]*']
-single_quoted = "'[^']+'"
-double_quoted = '"[^"]+"'
-non_space = '[^ "\'>]+'
-htmlstrip = re.compile("<" # open
- "([^>]+) " # prefix
- "(?:%s) *" % ('|'.join(bad_attrs),) + # undesirable attributes
- '= *(?:%s|%s|%s)' % (non_space, single_quoted, double_quoted) + # value
- "([^>]*)" # postfix
- ">" # end
-, re.I)
-def clean_attributes(html):
- while htmlstrip.search(html):
- html = htmlstrip.sub('<\\1\\2>', html)
- return html
-
diff --git a/readability/readability.py b/readability/readability.py
index 9c5f5c6c..c5739056 100755
--- a/readability/readability.py
+++ b/readability/readability.py
@@ -1,343 +1,747 @@
#!/usr/bin/env python
-from BeautifulSoup import NavigableString
-from page_parser import parse, get_title, get_body, Unparseable
import logging
import re
-
-REGEXES = { 'unlikelyCandidatesRe': re.compile('combx|comment|disqus|foot|header|menu|meta|nav|rss|shoutbox|sidebar|sponsor',re.I),
- 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main',re.I),
- 'positiveRe': re.compile('article|body|content|entry|hentry|page|pagination|post|text',re.I),
- 'negativeRe': re.compile('combx|comment|contact|foot|footer|footnote|link|media|meta|promo|related|scroll|shoutbox|sponsor|tags|widget',re.I),
- 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I),
- 'replaceBrsRe': re.compile('( ]*>[ \n\r\t]*){2,}',re.I),
- 'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I),
- 'trimRe': re.compile('^\s+|\s+$/'),
- 'normalizeRe': re.compile('\s{2,}/'),
- 'killBreaksRe': re.compile('( (\s| ?)*){1,}/'),
- 'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I),
+import sys
+import urllib.request
+import urllib.parse
+import urllib.error
+
+from lxml.etree import tostring
+from lxml.etree import tounicode
+from lxml.etree import _ElementTree
+from lxml.html import document_fromstring
+from lxml.html import fragment_fromstring
+from lxml.html import HtmlElement
+
+from .cleaners import clean_attributes
+from .cleaners import html_cleaner
+from .htmls import build_doc
+from .htmls import get_body
+from .htmls import get_title
+from .htmls import get_author
+from .htmls import shorten_title
+from .debug import describe, text_content
+
+
+log = logging.getLogger("readability.readability")
+
+REGEXES = {
+ "unlikelyCandidatesRe": re.compile(
+ r"combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter",
+ re.I,
+ ),
+ "okMaybeItsACandidateRe": re.compile(r"and|article|body|column|main|shadow", re.I),
+ "positiveRe": re.compile(
+ r"article|body|content|entry|hentry|main|page|pagination|post|text|blog|story",
+ re.I,
+ ),
+ "negativeRe": re.compile(
+ r"combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget",
+ re.I,
+ ),
+ "divToPElementsRe": re.compile(
+ r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", re.I
+ ),
+ # 'replaceBrsRe': re.compile(r'( ]*>[ \n\r\t]*){2,}',re.I),
+ # 'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I),
+ # 'trimRe': re.compile(r'^\s+|\s+$/'),
+ # 'normalizeRe': re.compile(r'\s{2,}/'),
+ # 'killBreaksRe': re.compile(r'( (\s| ?)*){1,}/'),
+ "videoRe": re.compile(r"https?:\/\/(www\.)?(youtube|vimeo)\.com", re.I),
+ # skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
}
-from collections import defaultdict
-def describe(node):
- if not hasattr(node, 'name'):
- return "[text]"
- return "%s#%s.%s" % (
- node.name, node.get('id', ''), node.get('class',''))
-def _text(node):
- return " ".join(node.findAll(text=True))
+class Unparseable(ValueError):
+ pass
+
+
+def to_int(x):
+ if not x:
+ return None
+ x = x.strip()
+ if x.endswith("px"):
+ return int(x[:-2])
+ if x.endswith("em"):
+ return int(x[:-2]) * 12
+ return int(x)
+
+
+def clean(text):
+ # Many spaces make the following regexes run forever
+ text = re.sub(r"\s{255,}", " " * 255, text)
+ text = re.sub(r"\s*\n\s*", "\n", text)
+ text = re.sub(r"\t|[ \t]{2,}", " ", text)
+ return text.strip()
+
+
+def text_length(i):
+ return len(clean(i.text_content() or ""))
+
+
+def compile_pattern(elements):
+ if not elements:
+ return None
+ elif isinstance(elements, re.Pattern):
+ return elements
+ elif isinstance(elements, (str, bytes)):
+ if isinstance(elements, bytes):
+ elements = str(elements, "utf-8")
+ elements = elements.split(",")
+ if isinstance(elements, (list, tuple)):
+ return re.compile("|".join([re.escape(x.strip()) for x in elements]), re.U)
+ else:
+ raise Exception(f"Unknown type for the pattern: {type(elements)}")
+ # assume string or string like object
+
class Document:
- TEXT_LENGTH_THRESHOLD = 25
- RETRY_LENGTH = 250
-
- def __init__(self, input, notify=None, **options):
- self.input = input
- self.options = defaultdict(lambda: None)
- for k, v in options.items():
- self.options[k] = v
- self.notify = notify or logging.info
- self.html = None
-
- def _html(self, force=False):
- if force or self.html is None:
- self.html = parse(self.input, self.options['url'], notify=self.notify)
- return self.html
-
- def content(self):
- return get_body(self._html())
-
- def title(self):
- return get_title(self._html())
-
- def summary(self):
- try:
- ruthless = True
- while True:
- self._html(True)
- [i.extract() for i in self.tags(self.html, 'script', 'style')]
-
- if ruthless: self.remove_unlikely_candidates()
- self.transform_misused_divs_into_paragraphs()
- candidates = self.score_paragraphs(self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD))
- best_candidate = self.select_best_candidate(candidates)
- if best_candidate:
- article = self.get_article(candidates, best_candidate)
- else:
- if ruthless:
- ruthless = False
- self.debug("ended up stripping too much - going for a safer parse")
- # try again
- continue
- else:
- article = self.html.find('body') or self.html
-
- cleaned_article = self.sanitize(article, candidates)
- of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH)
- if ruthless and not of_acceptable_length:
- ruthless = False
- continue # try again
- else:
- return cleaned_article
- except StandardError, e:
- logging.exception('error getting summary:')
- raise Unparseable(str(e))
-
- def get_article(self, candidates, best_candidate):
- # Now that we have the top candidate, look through its siblings for content that might also be related.
- # Things like preambles, content split by ads that we removed, etc.
-
- sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2])
- output = parse("")
- for sibling in best_candidate['elem'].parent.contents:
- if isinstance(sibling, NavigableString): continue
- append = False
- if sibling is best_candidate['elem']:
- append = True
- sibling_key = HashableElement(sibling)
- if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold:
- append = True
-
- if sibling.name == "p":
- link_density = self.get_link_density(sibling)
- node_content = sibling.string or ""
- node_length = len(node_content)
-
- if node_length > 80 and link_density < 0.25:
- append = True
- elif node_length < 80 and link_density == 0 and re.search('\.( |$)', node_content):
- append = True
-
- if append:
- output.append(sibling)
-
- if not output: output.append(best_candidate)
- return output
-
- def select_best_candidate(self, candidates):
- sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True)
- self.debug("Top 5 candidates:")
- for candidate in sorted_candidates[:5]:
- elem = candidate['elem']
- self.debug("Candidate %s with score %s" % (describe(elem), candidate['content_score']))
-
- if len(sorted_candidates) == 0:
- return None
- best_candidate = sorted_candidates[0]
- self.debug("Best candidate %s with score %s" % (describe(best_candidate['elem']), best_candidate['content_score']))
- return best_candidate
-
- def get_link_density(self, elem):
- link_length = len("".join([i.text or "" for i in elem.findAll("a")]))
- text_length = len(_text(elem))
- return float(link_length) / max(text_length, 1)
-
- def score_paragraphs(self, min_text_length):
- candidates = {}
- elems = self.tags(self.html, "p","td")
-
- for elem in elems:
- parent_node = elem.parent
- grand_parent_node = parent_node.parent
- parent_key = HashableElement(parent_node)
- grand_parent_key = HashableElement(grand_parent_node)
-
- inner_text = _text(elem)
-
- # If this paragraph is less than 25 characters, don't even count it.
- if (not inner_text) or len(inner_text) < min_text_length:
- continue
-
- if parent_key not in candidates:
- candidates[parent_key] = self.score_node(parent_node)
- if grand_parent_node and grand_parent_key not in candidates:
- candidates[grand_parent_key] = self.score_node(grand_parent_node)
-
- content_score = 1
- content_score += len(inner_text.split(','))
- content_score += min([(len(inner_text) / 100), 3])
-
- candidates[parent_key]['content_score'] += content_score
- if grand_parent_node:
- candidates[grand_parent_key]['content_score'] += content_score / 2.0
-
- # Scale the final candidates score based on link density. Good content should have a
- # relatively small link density (5% or less) and be mostly unaffected by this operation.
- for elem, candidate in candidates.items():
- candidate['content_score'] *= (1 - self.get_link_density(elem))
- self.debug("candidate %s scored %s" % (describe(elem), candidate['content_score']))
-
- return candidates
-
- def class_weight(self, e):
- weight = 0
- if e.get('class', None):
- if REGEXES['negativeRe'].search(e['class']):
- weight -= 25
-
- if REGEXES['positiveRe'].search(e['class']):
- weight += 25
-
- if e.get('id', None):
- if REGEXES['negativeRe'].search(e['id']):
- weight -= 25
-
- if REGEXES['positiveRe'].search(e['id']):
- weight += 25
-
- return weight
-
- def score_node(self, elem):
- content_score = self.class_weight(elem)
- name = elem.name.lower()
- if name == "div":
- content_score += 5
- elif name == "blockquote":
- content_score += 3
- elif name == "form":
- content_score -= 3
- elif name == "th":
- content_score -= 5
- return { 'content_score': content_score, 'elem': elem }
-
- def debug(self, *a):
- if self.options['debug']:
- logging.debug(*a)
-
- def remove_unlikely_candidates(self):
- for elem in self.html.findAll():
- s = "%s%s" % (elem.get('class', ''), elem.get('id', ''))
- if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.name != 'body':
- self.debug("Removing unlikely candidate - %s" % (s,))
- elem.extract()
-
- def transform_misused_divs_into_paragraphs(self):
- for elem in self.html.findAll():
- if elem.name.lower() == "div":
- # transform
s that do not contain other block elements into
s
- if REGEXES['divToPElementsRe'].search(''.join(map(unicode, elem.contents))):
- self.debug("Altering div(#%s.%s) to p" % (elem.get('id', ''), elem.get('class', '')))
- elem.name = "p"
-
- def tags(self, node, *tag_names):
- for tag_name in tag_names:
- for e in node.findAll(tag_name):
- yield e
-
- def sanitize(self, node, candidates):
- for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"):
- if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: header.extract()
-
- for elem in self.tags(node, "form", "iframe"):
- elem.extract()
-
- # Conditionally clean
s,
s, and
s
- for el in self.tags(node, "table", "ul", "div"):
- weight = self.class_weight(el)
- el_key = HashableElement(el)
- if el_key in candidates:
- content_score = candidates[el_key]['content_score']
- else:
- content_score = 0
- name = el.name
-
- if weight + content_score < 0:
- el.extract()
- self.debug("Conditionally cleaned %s with weight %s and content score %s because score + content score was less than zero." %
- (describe(el), weight, content_score))
- elif len(_text(el).split(",")) < 10:
- counts = {}
- for kind in ['p', 'img', 'li', 'a', 'embed', 'input']:
- counts[kind] = len(el.findAll(kind))
- counts["li"] -= 100
-
- content_length = len(_text(el)) # Count the text length excluding any surrounding whitespace
- link_density = self.get_link_density(el)
- to_remove = False
- reason = ""
-
- if counts["img"] > counts["p"]:
- reason = "too many images"
- to_remove = True
- elif counts["li"] > counts["p"] and name != "ul" and name != "ol":
- reason = "more