diff --git a/.gitignore b/.gitignore index fc9543d..9735f13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,22 @@ PythonExtensionPatterns.bbprojectd/ -build/ + *.so +build/ doc/sphinx/build/ + __pycache__/ + .DS_Store +.idea + +cmake-build-debug/ +cmake-build-release/ + +cPyExtPatt.egg-info/ + +dist/ + +cPyExtPatt/ + +tmp/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..ce861ca --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,26 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/sphinx/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt + +formats: + - pdf + - epub diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..696c13b --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,151 @@ +cmake_minimum_required(VERSION 3.24) +project(PythonExtensionPatterns) + +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) + + +# /Library/Frameworks/Python.framework/Versions/3.11/include/python3.11/cpython/pymem.h:42:39: fatal error: commas at the end of enumerator lists are a C++11 extension [-Wc++11-extensions] +# PYMEM_ALLOCATOR_PYMALLOC_DEBUG = 6, +# ^ + +add_compile_options( + "-Wall" + "-Wextra" + "-Wpedantic" + "-Werror" + "-Wfatal-errors" + "-Wno-unused-variable" # Temporary + "-Wno-unused-parameter" # Temporary + "-fexceptions" + # To allow designated initialisers. + "-Wno-c99-extensions" + "-Wno-c++11-extensions" + "$<$:-O0;-g3;-ggdb>" + # Temporary + -Wno-unused-function +) + +function(dump_cmake_variables) + message(STATUS "==== dump_cmake_variables()") + get_cmake_property(_variableNames VARIABLES) + list (SORT _variableNames) + foreach (_variableName ${_variableNames}) + if (ARGV0) + unset(MATCHED) + string(REGEX MATCH ${ARGV0} MATCHED ${_variableName}) + if (NOT MATCHED) + continue() + endif() + endif() + message(STATUS "${_variableName}=${${_variableName}}") + endforeach() + message(STATUS "==== dump_cmake_variables() DONE") +endfunction() + + + +FIND_PACKAGE (Python3 3.13 EXACT REQUIRED COMPONENTS Interpreter Development) +#FIND_PACKAGE(PythonLibs 3.11 EXACT REQUIRED) +#SET(PythonLibs_DIR "/Library/Frameworks/Python.framework/Versions/3.8") +#FIND_PACKAGE(PythonLibs 3.8 REQUIRED PATHS ("/Library/Frameworks/Python.framework/Versions/3.8")) +#FindPythonLibs() +IF (Python3_FOUND) + INCLUDE_DIRECTORIES("${Python3_INCLUDE_DIRS}") +# get_filename_component(PYTHON_LINK_DIRECTORY ${PYTHON_LIBRARY} DIRECTORY) + # See: https://cmake.org/cmake/help/latest/module/FindPython3.html#module:FindPython3 + message("Python3_VERSION: ${Python3_VERSION}") + message("Python3_EXECUTABLE: ${Python3_EXECUTABLE}") + message("Python3_INTERPRETER_ID: ${Python3_INTERPRETER_ID}") + message("Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}") + message("Python3_STDLIB: ${Python3_STDLIB}") + message("Python3_STDARCH: ${Python3_STDARCH}") + message("Python3_LINK_OPTIONS: ${Python3_LINK_OPTIONS}") + message("Python3_LIBRARIES: ${Python3_LIBRARIES}") +ELSE () + MESSAGE(FATAL_ERROR "Unable to find Python libraries.") +ENDIF () + +include_directories( + src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList + src/cpy + src/cpy/Containers + src/cpy/Watchers +) + +add_executable(PythonExtensionPatterns + src/main.c +# PythonExtensionPatterns/PythonExtensionPatterns/main.c +# PythonExtensionPatterns/PythonExtensionPatterns/PythonExtensionPatterns.c + src/cpy/Exceptions/cExceptions.c + src/cpy/ModuleGlobals/cModuleGlobals.c + src/cpy/Object/cObject.c + src/cpy/ParseArgs/cParseArgs.c + src/cpy/RefCount/cPyRefs.c + # TODO: +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/main.c +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/py_call_super.c +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/py_call_super.h +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/py_import_call_execute.c +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/py_import_call_execute.h +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/SubclassList.c +# src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/SubclassList.h + src/cCanonical.c + src/scratch.c + # Legacy code. Removed in the documentation for version 0.2. + src/cpy/ParseArgs/cParseArgsHelper.cpp + src/cpy/Pickle/cCustomPickle.c + src/cpy/File/cFile.cpp + src/cpy/File/PythonFileWrapper.h + src/cpy/File/PythonFileWrapper.cpp + src/cpy/Capsules/spam.c + src/cpy/Capsules/spam_capsule.h + src/cpy/Capsules/spam_capsule.c + src/cpy/Capsules/spam_client.c + src/cpy/Capsules/datetimetz.c + src/cpy/cpp/placement_new.cpp + src/cpy/cpp/cUnicode.cpp + src/cpy/SimpleExample/cFibA.h + src/cpy/SimpleExample/cFibA.c +# src/cpy/SimpleExample/cFibB.c + src/cpy/Util/py_call_super.h + src/cpy/Util/py_call_super.c + src/cpy/Iterators/cIterator.c + src/cpy/Threads/cThreadLock.h + src/cpy/SubClass/sublist.c + src/cpy/Threads/cppsublist.cpp + src/cpy/Threads/csublist.c + src/cpy/Logging/cLogging.c + src/cpy/RefCount/cRefCount.c + src/cpy/Util/py_call_super.cpp + src/cpy/CtxMgr/cCtxMgr.c + src/cpy/Containers/DebugContainers.c + src/cpy/Containers/DebugContainers.h + src/cpy/StructSequence/cStructSequence.c + src/cpy/Watchers/DictWatcher.c + src/cpy/Watchers/DictWatcher.h + src/cpy/pyextpatt_util.c + src/cpy/pyextpatt_util.h + src/cpy/Watchers/cWatchers.c + src/cpy/Object/cSeqObject.c +) + +#link_directories(${PYTHON_LINK_LIBRARY}) + +#target_link_libraries(${PROJECT_NAME} ${PYTHON_LIBRARY}) + +link_directories(${Python3_LIBRARIES}) + +#target_link_libraries(${PROJECT_NAME} ${PYTHON_LIBRARY}) +target_link_libraries(${PROJECT_NAME} ${Python3_LIBRARIES}) + +MESSAGE(STATUS "Build type: " ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "Library Type: " ${LIB_TYPE}) +MESSAGE(STATUS "Compiler flags:" ${CMAKE_CXX_COMPILE_FLAGS}) +MESSAGE(STATUS "Compiler cxx debug flags:" ${CMAKE_CXX_FLAGS_DEBUG}) +MESSAGE(STATUS "Compiler cxx release flags:" ${CMAKE_CXX_FLAGS_RELEASE}) +MESSAGE(STATUS "Compiler cxx min size flags:" ${CMAKE_CXX_FLAGS_MINSIZEREL}) +MESSAGE(STATUS "Compiler cxx flags:" ${CMAKE_CXX_FLAGS}) + +dump_cmake_variables() diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..acd3fca --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,98 @@ +===================== +History +===================== + +0.3.0 (2025-03-20) +===================== + +Added Chapters +-------------- + +- "Containers and Reference Counts" which corrects the Python documentation where that is wrong, misleading or missing. +- "Struct Sequences (namedtuple in C)" which corrects the Python documentation where that is wrong, misleading or missing. +- "Context Managers" with practical C code examples. +- "Watchers" with practical examples for dictionary watchers (Python 3.12+). +- "Installation" for the project. +- "Source Code Layout" for the project. + +Changed Chapters +---------------- + +- Update the "Homogeneous Python Containers and C++" chapter. +- Expand the "Memory Leaks" chapter. +- Extended the "Logging" chapter to show how to access the CPython Frame from C. +- Add "Emulating Sequence Types" to the "Creating New Types" chapter. +- Expand the Index. + +Other +------ + +- Python versions supported: 3.9, 3.10, 3.11, 3.12, 3.13. +- Development Status :: 5 - Production/Stable +- The documentation content, example and test code has roughly doubled since version 0.2.2. +- PDF Documentation is 339 pages. + +TODO +---- + +- Add "Debugging Python with CLion". + +.. + .. todo:: + + Update this history file. + +0.2.2 (2024-10-21) +===================== + +- Expand note on PyDict_SetItem(), PySet_Add() with code in src/cpy/RefCount/cRefCount.c and tests. + +0.2.1 (2024-07-29) +===================== + +- Python versions supported: 3.9, 3.10, 3.11, 3.12, 3.13 (possibly backwards compatible with Python 3.6, 3.7, 3.8) +- Almost all example code is built and tested against these Python versions. +- Added a chapter on managing file paths and files between Python and C. +- Added a chapter on subclassing from your classes or builtin classes. +- Added a chapter on pickling from C. +- Added a chapter on Capsules. +- Added a chapter on Iterators and Generators. +- Added a chapter on memory leaks and how to detect them. +- Added a chapter on thread safety. +- Update "Homogeneous Python Containers and C++" to refer to https://github.com/paulross/PyCppContainers +- All the documentation has been extensively reviewed and corrected where necessary. +- Development Status :: 5 - Production/Stable + +Contributors +------------------------- + +Many thanks! + +Pull Requests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- https://github.com/marioemmanuel +- https://github.com/miurahr +- https://github.com/gdevanla +- https://github.com/joelwhitehouse +- https://github.com/dhermes +- https://github.com/gst +- https://github.com/adamchainz +- https://github.com/nnathan + + +Issues +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- https://github.com/ngoldbaum +- https://github.com/niki-sp +- https://github.com/ldo +- https://github.com/1a1a11a +- https://github.com/congma + +0.1.0 (2014-09-09) +===================== + +- First release. +- Originally "Examples of reliable coding of Python 'C' extensions by Paul Ross.". +- Development Status :: 3 - Alpha diff --git a/INSTALL.rst b/INSTALL.rst new file mode 100644 index 0000000..76745e7 --- /dev/null +++ b/INSTALL.rst @@ -0,0 +1,67 @@ +Installation +======================== + +This is primarily a documentation project hosted on +`Read the Docs `_. +However all the example code is buildable and testable so if you want to examine that then here is how to get the +project. + +From PyPi +------------------------ + +.. code-block:: console + + pip install cPyExtPatt + +From Source +------------------------ + +Choose a directory of your choice, in this case: ``~/dev/tmp``. + +.. code-block:: console + + mkdir -p ~/dev/tmp + cd ~/dev/tmp + git clone https://github.com/paulross/PythonExtensionPatterns.git + cd PythonExtensionPatterns + +Virtual Environment +--------------------- + +Create a Python environment in the directory of your choice, in this case: +``~/dev/tmp/PythonExtensionPatterns/venv_3.11`` and activate it: + +.. code-block:: console + + python3.11 -m venv venv_3.11 + source venv_3.11/bin/activate + + +Install the Dependencies +--------------------------------- + +.. code-block:: console + + pip install -r requirements.txt + +Running the Tests +----------------------- + +You now should be able to run the following commands successfully in +``~/dev/tmp/PythonExtensionPatterns`` with your environment activated: + +.. code-block:: console + + pytest tests/ + +Building the Documentation +---------------------------------- + +This will build the html and PDF documentation (requires a latex installation): + +.. code-block:: console + + cd doc/sphinx + make html latexpdf + + diff --git "a/Introduction \342\200\224 Python Extension Patterns 0.1.0 documentation.pdf" "b/Introduction \342\200\224 Python Extension Patterns 0.1.0 documentation.pdf" deleted file mode 100644 index 50d4975..0000000 Binary files "a/Introduction \342\200\224 Python Extension Patterns 0.1.0 documentation.pdf" and /dev/null differ diff --git a/LICENSE b/LICENSE.txt similarity index 94% rename from LICENSE rename to LICENSE.txt index f3b373d..7ad9e77 100644 --- a/LICENSE +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Paul Ross https://github.com/paulross +Copyright (c) 2014-2025 Paul Ross https://github.com/paulross Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e5bf035 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +graft type_objects +graft tests + diff --git a/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/engun.xcuserdatad/UserInterfaceState.xcuserstate b/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/engun.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 0000000..7e8f029 Binary files /dev/null and b/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/engun.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate b/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate index 8e5594d..ba00981 100644 Binary files a/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate and b/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/xcuserdata/engun.xcuserdatad/xcschemes/xcschememanagement.plist b/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/xcuserdata/engun.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000..769fd5a --- /dev/null +++ b/PythonExtensionPatterns/PythonExtensionPatterns.xcodeproj/xcuserdata/engun.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,14 @@ + + + + + SchemeUserState + + PythonExtensionPatterns.xcscheme_^#shared#^_ + + orderHint + 0 + + + + diff --git a/PythonExtensionPatterns/PythonExtensionPatterns/PythonExtensionPatterns.1 b/PythonExtensionPatterns/PythonExtensionPatterns/PythonExtensionPatterns.1 index c8583d3..f16f794 100644 --- a/PythonExtensionPatterns/PythonExtensionPatterns/PythonExtensionPatterns.1 +++ b/PythonExtensionPatterns/PythonExtensionPatterns/PythonExtensionPatterns.1 @@ -1,79 +1,79 @@ -.\"Modified from man(1) of FreeBSD, the NetBSD mdoc.template, and mdoc.samples. -.\"See Also: -.\"man mdoc.samples for a complete listing of options -.\"man mdoc for the short list of editing options -.\"/usr/share/misc/mdoc.template -.Dd 07/05/2014 \" DATE -.Dt PythonExtensionPatterns 1 \" Program name and manual section number -.Os Darwin -.Sh NAME \" Section Header - required - don't modify -.Nm PythonExtensionPatterns, -.\" The following lines are read in generating the apropos(man -k) database. Use only key -.\" words here as the database is built based on the words here and in the .ND line. -.Nm Other_name_for_same_program(), -.Nm Yet another name for the same program. -.\" Use .Nm macro to designate other names for the documented program. -.Nd This line parsed for whatis database. -.Sh SYNOPSIS \" Section Header - required - don't modify -.Nm -.Op Fl abcd \" [-abcd] -.Op Fl a Ar path \" [-a path] -.Op Ar file \" [file] -.Op Ar \" [file ...] -.Ar arg0 \" Underlined argument - use .Ar anywhere to underline -arg2 ... \" Arguments -.Sh DESCRIPTION \" Section Header - required - don't modify -Use the .Nm macro to refer to your program throughout the man page like such: -.Nm -Underlining is accomplished with the .Ar macro like this: -.Ar underlined text . -.Pp \" Inserts a space -A list of items with descriptions: -.Bl -tag -width -indent \" Begins a tagged list -.It item a \" Each item preceded by .It macro -Description of item a -.It item b -Description of item b -.El \" Ends the list -.Pp -A list of flags and their descriptions: -.Bl -tag -width -indent \" Differs from above in tag removed -.It Fl a \"-a flag as a list item -Description of -a flag -.It Fl b -Description of -b flag -.El \" Ends the list -.Pp -.\" .Sh ENVIRONMENT \" May not be needed -.\" .Bl -tag -width "ENV_VAR_1" -indent \" ENV_VAR_1 is width of the string ENV_VAR_1 -.\" .It Ev ENV_VAR_1 -.\" Description of ENV_VAR_1 -.\" .It Ev ENV_VAR_2 -.\" Description of ENV_VAR_2 -.\" .El -.Sh FILES \" File used or created by the topic of the man page -.Bl -tag -width "/Users/joeuser/Library/really_long_file_name" -compact -.It Pa /usr/share/file_name -FILE_1 description -.It Pa /Users/joeuser/Library/really_long_file_name -FILE_2 description -.El \" Ends the list -.\" .Sh DIAGNOSTICS \" May not be needed -.\" .Bl -diag -.\" .It Diagnostic Tag -.\" Diagnostic informtion here. -.\" .It Diagnostic Tag -.\" Diagnostic informtion here. -.\" .El -.Sh SEE ALSO -.\" List links in ascending order by section, alphabetically within a section. -.\" Please do not reference files that do not exist without filing a bug report -.Xr a 1 , -.Xr b 1 , -.Xr c 1 , -.Xr a 2 , -.Xr b 2 , -.Xr a 3 , -.Xr b 3 -.\" .Sh BUGS \" Document known, unremedied bugs +.\"Modified from man(1) of FreeBSD, the NetBSD mdoc.template, and mdoc.samples. +.\"See Also: +.\"man mdoc.samples for a complete listing of options +.\"man mdoc for the short list of editing options +.\"/usr/share/misc/mdoc.template +.Dd 07/05/2014 \" DATE +.Dt PythonExtensionPatterns 1 \" Program name and manual section number +.Os Darwin +.Sh NAME \" Section Header - required - don't modify +.Nm PythonExtensionPatterns, +.\" The following lines are read in generating the apropos(man -k) database. Use only key +.\" words here as the database is built based on the words here and in the .ND line. +.Nm Other_name_for_same_program(), +.Nm Yet another name for the same program. +.\" Use .Nm macro to designate other names for the documented program. +.Nd This line parsed for whatis database. +.Sh SYNOPSIS \" Section Header - required - don't modify +.Nm +.Op Fl abcd \" [-abcd] +.Op Fl a Ar path \" [-a path] +.Op Ar file \" [file] +.Op Ar \" [file ...] +.Ar arg0 \" Underlined argument - use .Ar anywhere to underline +arg2 ... \" Arguments +.Sh DESCRIPTION \" Section Header - required - don't modify +Use the .Nm macro to refer to your program throughout the man page like such: +.Nm +Underlining is accomplished with the .Ar macro like this: +.Ar underlined text . +.Pp \" Inserts a space +A list of items with descriptions: +.Bl -tag -width -indent \" Begins a tagged list +.It item a \" Each item preceded by .It macro +Description of item a +.It item b +Description of item b +.El \" Ends the list +.Pp +A list of flags and their descriptions: +.Bl -tag -width -indent \" Differs from above in tag removed +.It Fl a \"-a flag as a list item +Description of -a flag +.It Fl b +Description of -b flag +.El \" Ends the list +.Pp +.\" .Sh ENVIRONMENT \" May not be needed +.\" .Bl -tag -width "ENV_VAR_1" -indent \" ENV_VAR_1 is width of the string ENV_VAR_1 +.\" .It Ev ENV_VAR_1 +.\" Description of ENV_VAR_1 +.\" .It Ev ENV_VAR_2 +.\" Description of ENV_VAR_2 +.\" .El +.Sh FILES \" File used or created by the topic of the man page +.Bl -tag -width "/Users/joeuser/Library/really_long_file_name" -compact +.It Pa /usr/share/file_name +FILE_1 description +.It Pa /Users/joeuser/Library/really_long_file_name +FILE_2 description +.El \" Ends the list +.\" .Sh DIAGNOSTICS \" May not be needed +.\" .Bl -diag +.\" .It Diagnostic Tag +.\" Diagnostic informtion here. +.\" .It Diagnostic Tag +.\" Diagnostic informtion here. +.\" .El +.Sh SEE ALSO +.\" List links in ascending order by section, alphabetically within a section. +.\" Please do not reference files that do not exist without filing a bug report +.Xr a 1 , +.Xr b 1 , +.Xr c 1 , +.Xr a 2 , +.Xr b 2 , +.Xr a 3 , +.Xr b 3 +.\" .Sh BUGS \" Document known, unremedied bugs .\" .Sh HISTORY \" Document history if command behaves in a unique manner \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 3bc99c4..0000000 --- a/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# PythonExtensionPatterns - -Examples of reliable coding of Python 'C' extensions by Paul Ross. - -The full documentation is on [Read the Docs](http://pythonextensionpatterns.readthedocs.org/en/latest/index.html). - -Code examples and documentation source are right here on GitHub. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e97d250 --- /dev/null +++ b/README.rst @@ -0,0 +1,64 @@ +*************************** +PythonExtensionPatterns +*************************** + +If you need to write C extension for Python then this is the place for you. + +The full documentation is on +`Read the Docs `_. + +Code examples and documentation source are `on GitHub `_. +The example and test code is available `on PyPi `_. + +==================================== +Subjects Covered +==================================== + +- Introduction +- A Simple Example +- PyObjects and Reference Counting +- Containers and Reference Counts +- Struct Sequence Objects (a namedtuple in C) +- Exception Raising +- A Pythonic Coding Pattern for C Functions +- Parsing Python Arguments +- Creating New Types +- Setting and Getting Module Globals +- Logging and Frames +- File Paths and Files +- Subclassing and Using super() +- Capsules +- Iterators and Generators +- Context Managers +- Pickling C Extension Types +- Watchers [Python 3.12+] +- Setting Compiler Flags +- Debugging +- Memory Leaks +- Thread Safety +- Source Code Layout +- Using C++ With CPython Code +- Miscellaneous +- Installation +- Further Reading +- TODO +- History +- Index + +============= +Project Links +============= + +- Source is `on GitHub `_. +- Documentation `on Read the Docs `_. +- Project is `on PyPi `_. + +================== +Videos +================== + +I have presented some of this, well mostly the chapter "PyObjects and Reference Counting", +at Python conferences so if you prefer videos they are here: + +- `PyCon UK 2015 `_ +- `PyCon US 2016 `_ diff --git a/build_all.sh b/build_all.sh new file mode 100755 index 0000000..64dee52 --- /dev/null +++ b/build_all.sh @@ -0,0 +1,268 @@ +#!/bin/bash +# +# Builds svfs for distribution +# Ref: https://packaging.python.org/tutorials/packaging-projects/ +# +# Other references: +# https://kvz.io/bash-best-practices.html +# https://bertvv.github.io/cheat-sheets/Bash.html + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +# For current versions see https://devguide.python.org/versions/ +# Takes about 70 seconds per version. +PYTHON_VERSIONS=('3.9' '3.10' '3.11' '3.12' '3.13') +# Used for venvs +PYTHON_VENV_ROOT="${HOME}/pyenvs" +PROJECT_NAME="PyExtPatt" +CPP_EXECUTABLE="PythonExtensionPatterns" + +usage() +{ + echo "usage: build_all.sh [-d] [-r] [-h, --help]" + echo "options:" + echo " -h, --help Print help and exit." + echo " -d Build documentation (slow)." + echo " -r Remove and rebuild all virtual environments." +} + +# If -h or --help print help. +for arg in "$@" +do + if [ "$arg" == "--help" ] || [ "$arg" == "-h" ] + then + usage + exit + fi +done + +OPT_REMOVE_REBUILD_VENVS=false +OPT_BUILD_DOCUMENTATION=false + +if [[ "$#" -gt 0 ]]; then +for arg in "$@" +do + case "$arg" in + -r) OPT_REMOVE_REBUILD_VENVS=true ;; # Remove existing venvs and rebuild them. + -d) OPT_BUILD_DOCUMENTATION=true ;; # Build documentation. + esac +done +fi + +#printf "%-8s %8s %10s %10s %12s\n" "Ext" "Files" "Lines" "Words" "Bytes" + +build_cpp() { + echo "---> C++ clean debug" + cmake --build cmake-build-debug --target clean -- -j 6 + echo "---> C++ build debug" + cmake --build cmake-build-debug --target ${CPP_EXECUTABLE} -- -j 6 +# echo "---> C++ clean release" +# cmake --build cmake-build-release --target clean -- -j 6 +# echo "---> C++ build release" +# cmake --build cmake-build-release --target ${CPP_EXECUTABLE} -- -j 6 +} + +run_cpp_tests() { + echo "---> C++ debug tests" + cmake-build-debug/${CPP_EXECUTABLE} +# echo "---> C++ release tests" +# cmake-build-release/${CPP_EXECUTABLE} +} + +deactivate_virtual_environment() { + # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo + set +u + if command -v deactivate &>/dev/null; then + deactivate + fi + set -u +} + +create_virtual_environments() { + deactivate_virtual_environment + for version in ${PYTHON_VERSIONS[*]}; do + echo "---> Create virtual environment for Python version ${version}" + venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" + if [ ! -d "${venv_path}" ]; then + # Control will enter here if directory not exists. + echo "---> Creating virtual environment at: ${venv_path}" + "python${version}" -m venv "${venv_path}" + fi + # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo + set +u + source "${venv_path}/bin/activate" + set -u + echo "---> Python version:" + python -VV + echo "---> Installing everything via pip:" + pip install -U pip setuptools wheel + pip install -r requirements.txt + # Needed for uploading to pypi + pip install twine + echo "---> Result of pip install:" + pip list + done +} + +remove_virtual_environments() { + deactivate_virtual_environment + for version in ${PYTHON_VERSIONS[*]}; do + echo "---> For Python version ${version}" + venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" + if [ -d "${venv_path}" ]; then + # Control will enter here if directory exists. + echo "---> Removing virtual environment at: ${venv_path}" + #rm --recursive --force -- "${venv_path}" + rm -rf -- "${venv_path}" + fi + done +} + +create_and_test_bdist_wheel() { + echo "---> Creating bdist_wheel for all versions..." + for version in ${PYTHON_VERSIONS[*]}; do + echo "---> For Python version ${version}" + deactivate_virtual_environment + venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" + if [ ! -d "${venv_path}" ]; then + # Control will enter here if directory doesn't exist. + echo "---> Creating virtual environment at: ${venv_path}" + "python${version}" -m venv "${venv_path}" + else + echo "---> EXISTING Virtual environment at: ${venv_path}" + fi + # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo + set +u + source "${venv_path}/bin/activate" + set -u + echo "---> Python version:" + python -VV +# echo "---> Installing everything via pip:" +# pip install -U pip setuptools wheel +# pip install -r requirements.txt +# # Needed for uploading to pypi +# pip install twine +# echo "---> Result of pip install:" +# pip list + echo "---> Running python setup.py develop:" +# MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py develop + python setup.py develop + echo "---> Running tests:" + # Fail fast with -x + pytest tests -x + # Run all tests (slow). +# pytest tests --runslow --benchmark-sort=name +# pytest tests -v + echo "---> Running setup for bdist_wheel:" + # Need wheel otherwise bdist_wheel "error: invalid command 'bdist_wheel'" + pip install wheel + python setup.py bdist_wheel + done +} + +create_sdist() { + echo "---> Running setup for sdist:" + python setup.py sdist +} + +create_documentation() { + echo "---> Python version:" + which python + python -VV + echo "---> pip list:" + pip list + echo "---> Copying files from project root to doc/sphinx/source:" + echo "---> Building documentation:" + cd doc/sphinx + rm -rf build/ + make html latexpdf + cd ../.. +# echo "---> Generating stub file:" +# python stubgen_simple.py +} + +report_all_versions_and_setups() { + echo "---> Reporting all versions..." + for version in ${PYTHON_VERSIONS[*]}; do + echo "---> For Python version ${version}" + deactivate_virtual_environment + venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" + if [ ! -d "${venv_path}" ]; then + # Control will enter here if directory doesn't exist. + echo "---> Creating virtual environment at: ${venv_path}" + "python${version}" -m venv "${venv_path}" + else + echo "---> EXISTING Virtual environment at: ${venv_path}" + fi + # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo + set +u + source "${venv_path}/bin/activate" + set -u + echo "---> Python version:" + python -VV + echo "---> pip list:" + pip list + done +# Don't do this as we want to use show_results_of_dist() and twine after this +# deactivate_virtual_environment +} + +show_results_of_dist() { + echo "---> dist/:" + ls -l "dist" + echo "---> twine check dist/*:" + pip install twine + twine check dist/* + # Test from Test PyPi + # pip install -i https://test.pypi.org/simple/orderedstructs + echo "---> Ready for upload to test PyPi:" + echo "---> pip install twine" + echo "---> twine upload --repository testpypi dist/*" + echo "---> Or PyPi:" + echo "---> twine upload dist/*" +} + +echo "===> Start date:" +date +echo "Options:" +echo "===> OPT_REMOVE_REBUILD_VENVS: $OPT_REMOVE_REBUILD_VENVS" +echo "===> OPT_BUILD_DOCUMENTATION: $OPT_BUILD_DOCUMENTATION" +# See: https://stackoverflow.com/questions/41150814/how-to-echo-all-values-from-array-in-bash +IFS="," +echo "===> PYTHON_VERSIONS: ${PYTHON_VERSIONS[*]}" + +echo "===> Clean and build C++ code" +build_cpp +echo "===> Running C++ tests" +run_cpp_tests +echo "===> Removing build/ and dist/" +#rm --recursive --force -- "build" "dist" +rm -rf -- "build" "dist" "cPyExtPatt" + +if [ $OPT_REMOVE_REBUILD_VENVS = true ]; then +echo "===> Removing virtual environments" +remove_virtual_environments +echo "===> Creating virtual environments" +create_virtual_environments +fi + +echo "===> Creating binary wheels" +create_and_test_bdist_wheel +echo "===> Creating source distribution" +create_sdist +echo "===> All versions and setups:" +report_all_versions_and_setups + +if [ $OPT_BUILD_DOCUMENTATION = true ]; then +echo "===> Building documentation:" +create_documentation +fi + +echo "===> dist/ result:" +show_results_of_dist +#deactivate_virtual_environment +echo "===> End date:" +date +echo "===> All done" diff --git a/doc/presentations/ReadMe.md b/doc/presentations/ReadMe.md new file mode 100644 index 0000000..cbeb52a --- /dev/null +++ b/doc/presentations/ReadMe.md @@ -0,0 +1,5 @@ +# Presentations about Python Extension Patterns + +PyCon UK 2015: [https://www.youtube.com/watch?v=ViRIYqiU128](https://www.youtube.com/watch?v=ViRIYqiU128) + +PyCon US 2016: [https://www.youtube.com/watch?v=Yq__HtUIH5Y](https://www.youtube.com/watch?v=Yq__HtUIH5Y) diff --git a/doc/sphinx/source/HISTORY.rst b/doc/sphinx/source/HISTORY.rst new file mode 100644 index 0000000..aff0dd8 --- /dev/null +++ b/doc/sphinx/source/HISTORY.rst @@ -0,0 +1,4 @@ +.. index:: + single: History + +.. include:: ../../../HISTORY.rst diff --git a/doc/sphinx/source/_headings.rst b/doc/sphinx/source/_headings.rst new file mode 100644 index 0000000..aa034a9 --- /dev/null +++ b/doc/sphinx/source/_headings.rst @@ -0,0 +1,94 @@ + +############### +PartOne +############### + +Section headers (ref) are created by underlining (and optionally overlining) the section title with a punctuation +character, at least as long as the text: + +Normally, there are no heading levels assigned to certain characters as the structure is determined from the succession +of headings. However, this convention is used in Python Developer’s Guide for documenting which you may follow: + +# with overline, for parts +* with overline, for chapters += for sections +- for subsections +^ for subsubsections +" for paragraphs + +``#`` with overline. + +Text for PartOne. + +*************************** +PartOne.ChapterOne +*************************** + +``*`` with overline. + +Text for PartOne.ChapterOne. + +====================================== +PartOne.ChapterOne.SectionOne +====================================== + +``=`` with overline. + +Text for PartOne.ChapterOne.SectionOne + +PartOne.ChapterOne.SectionOne +====================================== + +``=`` without overline. + +Text for PartOne.ChapterOne.SectionOne + +-------------------------------------------------- +PartOne.ChapterOne.SectionOne.SubsectionOne +-------------------------------------------------- + +``-`` with overline. + +PartOne.ChapterOne.SectionOne.SubsectionOne +-------------------------------------------------- + +``-`` without overline. + +Text for PartOne.ChapterOne.SectionOne.SubsectionOne + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``^`` with overline. + +Text for PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne + +PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``^`` without overline. + +Text for PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne + +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne.ParagraphOne +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``"`` with overline. + +Text for PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne.ParagraphOne + +PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne.ParagraphOne +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``"`` without overline. + +Text for PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne.ParagraphOne + +PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne.ParagraphTwo +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``"`` without overline. + +Text for PartOne.ChapterOne.SectionOne.SubsectionOne.SubsubsectionOne.ParagraphTwo diff --git a/doc/sphinx/source/_index_styles.rst b/doc/sphinx/source/_index_styles.rst new file mode 100644 index 0000000..adaab06 --- /dev/null +++ b/doc/sphinx/source/_index_styles.rst @@ -0,0 +1,116 @@ + +.. + Explore different index styles. + +====================================== +Index Styles +====================================== + +See: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-index + +.. + .. index:: + single: execution; context + pair: module; __main__ + pair: module; sys + triple: module; search; path + seealso: execution + +---------------------- +Inline Index Entries +---------------------- + +This is a normal reStructuredText :index:`paragraph` that contains several :index:`index entries `. + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. index:: + single: execution + +----------------- +Single execution +----------------- + +``single: execution``. + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. index:: + single: execution; context + +------------------------- +Single execution; context +------------------------- + +``single: execution; context``. + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. index:: + pair: module; __main__ + pair: module; sys + +----------------- +Pairs +----------------- + +``pair: module; __main__``. + +``pair: module; sys``. + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. index:: + triple: module; search; path + +----------------- +Triple +----------------- + +``triple: module; search; path`` + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. index:: + see: execution; context + +------------------------- +See execution; context +------------------------- + +``see: execution; context`` + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. index:: + seealso: execution; context + +------------------------------ +See Also execution; context +------------------------------ + +``seealso: execution; context`` diff --git a/doc/sphinx/source/canonical_function.rst b/doc/sphinx/source/canonical_function.rst index c7762b3..38abfdf 100644 --- a/doc/sphinx/source/canonical_function.rst +++ b/doc/sphinx/source/canonical_function.rst @@ -1,16 +1,24 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: C Functions; Coding Pattern + +.. _chapter_function_pattern: =========================================== A Pythonic Coding Pattern for C Functions =========================================== -To avoid all the errors we have seen it is useful to have a C coding pattern for handling ``PyObjects`` that does the following: +Principle +=============== + +To avoid all the errors we have seen, particularly in :ref:`chapter_refcount` it is useful to have a C coding pattern +for handling ``PyObjects`` that does the following: * No early returns and a single place for clean up code. * Borrowed references incref'd and decref'd correctly. -* No stale Exception from previous execution path. +* No stale Exception from a previous execution path. * Exceptions set and tested. * NULL is returned when an exception is set. * Non-NULL is returned when no exception is set. @@ -27,6 +35,9 @@ The basic pattern in C is similar to Python's try/except/finally pattern: /* Clean up under normal conditions and return an appropriate value. */ +Coding the Function +===================== + Firstly we set any local ``PyObject`` (s) and the return value to ``NULL``: .. code-block:: c @@ -48,15 +59,17 @@ Check that there are no lingering Exceptions: assert(! PyErr_Occurred()); -An alternative check for no lingering Exceptions: +An alternative check for no lingering Exceptions with non-debug builds: .. code-block:: c if(PyErr_Occurred()) { + PyObject_Print(PyErr_Occurred(), stdout, Py_PRINT_RAW); goto except; } -Now we assume that any argument is a "Borrowed" reference so we increment it (we need a matching ``Py_DECREF`` before function exit, see below). The first pattern assumes a non-NULL argument. +Now we assume that any argument is a "Borrowed" reference so we increment it (we need a matching ``Py_DECREF`` before +function exit, see below). The first pattern assumes a non-NULL argument. .. code-block:: c @@ -71,9 +84,10 @@ If you are willing to accept NULL arguments then this pattern would be more suit Py_INCREF(arg_1); } -Of course the same test must be used when calling ``Py_DECFREF``, or just use ``Py_XDECREF``. +Or just use ``Py_XINCREF``. -Now we create any local objects, if they are "Borrowed" references we need to incref them. With any abnormal behaviour we do a local jump straight to the cleanup code. +Now we create any local objects, if they are "Borrowed" references we need to incref them. +With any abnormal behaviour we do a local jump straight to the cleanup code. .. code-block:: c @@ -139,6 +153,17 @@ Notice the ``except:`` block falls through to the ``finally:`` block. return ret; } +.. index:: + single: C Functions; Full Coding Pattern + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +The Function Code as One +======================== Here is the complete code with minimal comments: @@ -154,7 +179,8 @@ Here is the complete code with minimal comments: assert(arg_1); Py_INCREF(arg_1); - /* obj_a = ...; */ + /* Create obj_a = ...; */ + if (! obj_a) { PyErr_SetString(PyExc_ValueError, "Ooops."); goto except; diff --git a/doc/sphinx/source/capsules.rst b/doc/sphinx/source/capsules.rst new file mode 100644 index 0000000..1a9b85d --- /dev/null +++ b/doc/sphinx/source/capsules.rst @@ -0,0 +1,743 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 2 + +.. _chapter_capsules: + +.. index:: + single: Capsules + +*************** +Capsules +*************** + +Usually C extension code is used solely by the extension itself, for that reason all functions are declared ``static`` +however there are cases where the C extension code is useful to another C extension. +When modules are used as shared libraries the symbols in one extension might not be visible to another extension. + +`Capsules `_ +are a means by which this can be achieved by passing C pointers from one extension module to another. + + +.. index:: + single: Capsules; Simple Example + +================================ +A Simple Example +================================ + +This takes the example given in the Python documentation and makes it complete. +The code is in ``src/cpy/Capsules/spam*``. + +--------------------------- +Basic Extension +--------------------------- + +Here is the basic example of an Extension that can make a system call. +The code is in ``src/cpy/Capsules/spam.c``. + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + #include + + static PyObject * + spam_system(PyObject *Py_UNUSED(self), PyObject *args) { + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); + } + + static PyMethodDef SpamMethods[] = { + /* ... */ + {"system", spam_system, METH_VARARGS, + "Execute a shell command."}, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static struct PyModuleDef spammodule = { + PyModuleDef_HEAD_INIT, + "spam", /* name of module */ + PyDoc_STR("Documentation for the spam module"), + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods, + NULL, NULL, NULL, NULL, + }; + + PyMODINIT_FUNC + PyInit_spam(void) { + return PyModule_Create(&spammodule); + } + +This would be built with this entry in ``setup.py``: + +.. code-block:: python + + Extension(f"{PACKAGE_NAME}.Capsules.spam", sources=['src/cpy/Capsules/spam.c',], + include_dirs=['/usr/local/include', ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +This can be tested with the code in ``tests/unit/test_c_capsules.py``: + +.. code-block:: python + + from cPyExtPatt.Capsules import spam + + def test_spam(): + result = spam.system("ls -l") + assert result == 0 + + +.. index:: + single: Capsules; Exporting + +--------------------------- +An Exportable Extension +--------------------------- + +To make a version that exports its API in a Capsule we will make some changes: + +- Introduce a function ``PySpam_System`` that actually does the system call. We will export this. +- Introduce a header file ``src/cpy/Capsules/spam_capsule.h`` that contains the exports. +- Introduce a macro ``SPAM_CAPSULE`` that controls whether we are exporting or importing the API. +- Change the module initialisation to initialise the API. + +The header file in ``src/cpy/Capsules/spam_capsule.h`` looks like this: + +.. code-block:: c + + #ifndef Py_SPAM_CAPSULE_H + #define Py_SPAM_CAPSULE_H + #ifdef __cplusplus + extern "C" { + #endif + + #include + + /* Header file for spammodule */ + + /* C API functions */ + #define PySpam_System_NUM 0 + #define PySpam_System_RETURN int + #define PySpam_System_PROTO (const char *command) + + /* Total number of C API pointers */ + #define PySpam_API_pointers 1 + + #ifdef SPAM_CAPSULE + + /* This section is used when compiling spam_capsule.c */ + static PySpam_System_RETURN PySpam_System PySpam_System_PROTO; + + #else + /* This section is used in modules that use spam_capsule's API */ + static void **PySpam_API; + + #define PySpam_System \ + (*(PySpam_System_RETURN (*)PySpam_System_PROTO) PySpam_API[PySpam_System_NUM]) + + /* Return -1 on error, 0 on success. + * PyCapsule_Import will set an exception if there's an error. + */ + static int + import_spam_capsule(void) { + PySpam_API = (void **)PyCapsule_Import("cPyExtPatt.Capsules.spam_capsule._C_API", 0); + return (PySpam_API != NULL) ? 0 : -1; + } + #endif + #ifdef __cplusplus + } + #endif + #endif /* !defined(Py_SPAM_CAPSULE_H) */ + +The full code is in ``src/cpy/Capsules/spam_capsule.c``: + +.. code-block:: c + + // Implements: https://docs.python.org/3/extending/extending.html#extending-simpleexample + // as a capsule: https://docs.python.org/3/extending/extending.html#providing-a-c-api-for-an-extension-module + // Includes specific exception. + // Lightly edited. + + #define PY_SSIZE_T_CLEAN + + #include + + #define SPAM_CAPSULE + + #include "spam_capsule.h" + + static int + PySpam_System(const char *command) { + return system(command); + } + + static PyObject * + spam_system(PyObject *Py_UNUSED(self), PyObject *args) { + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = PySpam_System(command); + return PyLong_FromLong(sts); + } + + static PyMethodDef SpamMethods[] = { + /* ... */ + {"system", spam_system, METH_VARARGS, + "Execute a shell command."}, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static struct PyModuleDef spammodule = { + PyModuleDef_HEAD_INIT, + "spam_capsule", /* name of module */ + PyDoc_STR("Documentation for the spam module"), + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods, + NULL, NULL, NULL, NULL, + }; + + PyMODINIT_FUNC + PyInit_spam_capsule(void) { + PyObject *m; + static void *PySpam_API[PySpam_API_pointers]; + PyObject *c_api_object; + + m = PyModule_Create(&spammodule); + if (m == NULL) + return NULL; + + /* Initialize the C API pointer array */ + PySpam_API[PySpam_System_NUM] = (void *) PySpam_System; + + /* Create a Capsule containing the API pointer array's address */ + c_api_object = PyCapsule_New((void *) PySpam_API, "cPyExtPatt.Capsules.spam_capsule._C_API", NULL); + + if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { + Py_XDECREF(c_api_object); + Py_DECREF(m); + return NULL; + } + return m; + } + +This can be built by adding this Extension to ``setup.py``: + +.. code-block:: python + + Extension(f"{PACKAGE_NAME}.Capsules.spam_capsule", + sources=['src/cpy/Capsules/spam_capsule.c',], + include_dirs=['/usr/local/include', 'src/cpy/Capsules',], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +This can be tested with the code in ``tests/unit/test_c_capsules.py``: + +.. code-block:: python + + from cPyExtPatt.Capsules import spam_capsule + + def test_spam_capsule(): + result = spam_capsule.system("ls -l") + assert result == 0 + +.. index:: + single: Capsules; Using Exported API + +--------------------------- +A Client Extension +--------------------------- + +This can now be used by another extension, say ``spam_client`` in ``src/cpy/Capsules/spam_client.c``. +Note the use of: + +- ``#include "spam_capsule.h"`` +- The call to the imported function ``PySpam_System``. +- Calling ``import_spam_capsule()`` in the module initialisation. + +Here is the complete C code: + +.. code-block:: c + + // Implements: https://docs.python.org/3/extending/extending.html#extending-simpleexample + // but using a capsule exported by spam_capsule.h/.c + // Excludes specific exception. + // Lightly edited. + + #define PY_SSIZE_T_CLEAN + + #include + #include "spam_capsule.h" + + static PyObject * + spam_system(PyObject *Py_UNUSED(self), PyObject *args) { + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = PySpam_System(command); + return PyLong_FromLong(sts); + } + + static PyMethodDef SpamMethods[] = { + /* ... */ + {"system", spam_system, METH_VARARGS, + "Execute a shell command."}, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static struct PyModuleDef spam_clientmodule = { + PyModuleDef_HEAD_INIT, + "spam_client", /* name of module */ + PyDoc_STR("Documentation for the spam module"), + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods, + NULL, NULL, NULL, NULL, + }; + + PyMODINIT_FUNC + PyInit_spam_client(void) { + PyObject *m; + + m = PyModule_Create(&spam_clientmodule); + if (m == NULL) + return NULL; + if (import_spam_capsule() < 0) + return NULL; + /* additional initialization can happen here */ + return m; + } + +This can be built by adding this Extension to ``setup.py``: + +.. code-block:: python + + Extension(f"{PACKAGE_NAME}.Capsules.spam_client", + sources=['src/cpy/Capsules/spam_client.c',], + include_dirs=['/usr/local/include', 'src/cpy/Capsules',], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +This can be tested with the code in ``tests/unit/test_c_capsules.py``: + +.. code-block:: python + + from cPyExtPatt.Capsules import spam_client + + def test_spam_client(): + result = spam_client.system("ls -l") + assert result == 0 + + +.. _chapter_capsules_using_an_existing_capsule: + +.. index:: + single: Capsules; Using Existing API + single: Subclassing; datetime Example + +================================ +Using an Existing Capsule +================================ + +Here is an example of using an existing Capsule, the ``datetime`` C API. +In this case we want to create a subclass of the ``datetime.datetime`` object that always has a time zone i.e. no +`naive` datetimes. + +Here is the C Extension code to create a ``datetimetz`` module and a ``datetimetz.datetimetz`` object. +This code is lightly edited for clarity and works with Python 3.10+ (using the modern API). +The actual code is in ``src/cpy/Capsules/datetimetz.c`` (which works with the Python 3.9 API as well) +and the tests are in ``tests/unit/test_c_capsules.py``. + +-------------------------------- +Writing the Code for the Object +-------------------------------- + +Firstly the declaration of the timezone aware datetime, it just inherits from ``datetime.datetime`` : + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + #include + #include "datetime.h" + + typedef struct { + PyDateTime_DateTime datetime; + } DateTimeTZ; + +Then create a function that sets an error if the ``DateTimeTZ`` lacks a tzinfo +This will be used in a couple of places. + +.. code-block:: c + + /* This function sets an error if a tzinfo is not set and returns NULL. + * In practice this would use Python version specific calls. + * For simplicity this uses Python 3.10+ code. + */ + static DateTimeTZ * + raise_if_no_tzinfo(DateTimeTZ *self) { + if (!_PyDateTime_HAS_TZINFO(&self->datetime)) { + PyErr_SetString(PyExc_TypeError, "No time zone provided."); + Py_DECREF(self); + self = NULL; + } + return self; + } + +Now the code for creating a new instance: + +.. code-block:: c + + static PyObject * + DateTimeTZ_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + DateTimeTZ *self = (DateTimeTZ *) PyDateTimeAPI->DateTimeType->tp_new( + type, args, kwds + ); + if (self) { + self = raise_if_no_tzinfo(self); + } + return (PyObject *) self; + } + +So far a new ``datetimetz`` object must be created with a ``tzinfo`` but the ``datetime.datetime`` has an API +``replace()`` that creates a new datetime with different properties, including ``tzinfo``. +We need to guard against the user trying to change the timezone to None. +To do this we call the super class function and then check, and raise, if a ``tzinfo`` is absent. +This uses the utility function that call Python's ``super()`` function. +That code is in ``src/cpy/Util/py_call_super.h`` and ``src/cpy/Util/py_call_super.c``: + +.. code-block:: c + + static PyObject * + DateTimeTZ_replace(PyObject *self, PyObject *args, PyObject *kwargs) { + PyObject *result = call_super_name(self, "replace", args, kwargs); + if (result) { + result = (PyObject *) raise_if_no_tzinfo((DateTimeTZ *) result); + } + return result; + } + +Finally the module code: + +.. code-block:: c + + static PyMethodDef DateTimeTZ_methods[] = { + { + "replace", + (PyCFunction) DateTimeTZ_replace, + METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Return datetime with new specified fields.") + }, + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static PyTypeObject DatetimeTZType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "datetimetz.datetimetz", + .tp_doc = "A datetime that requires a time zone.", + .tp_basicsize = sizeof(DateTimeTZ), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = DateTimeTZ_new, + .tp_methods = DateTimeTZ_methods, + }; + + static PyModuleDef datetimetzmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "datetimetz", + .m_doc = ( + "Module that contains a datetimetz," + "a datetime.datetime with a mandatory time zone." + ), + .m_size = -1, + }; + +Initialise the module, this is when we use the existing capsule: + +.. code-block:: c + + PyMODINIT_FUNC + PyInit_datetimetz(void) { + PyObject *m = PyModule_Create(&datetimetzmodule); + if (m == NULL) { + return NULL; + } + // datetime.datetime_CAPI + PyDateTime_IMPORT; + if (!PyDateTimeAPI) { + Py_DECREF(m); + return NULL; + } + // Set inheritance. + DatetimeTZType.tp_base = PyDateTimeAPI->DateTimeType; + if (PyType_Ready(&DatetimeTZType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&DatetimeTZType); + PyModule_AddObject(m, "datetimetz", (PyObject *) &DatetimeTZType); + /* additional initialization can happen here */ + return m; + } + +The extension is created with this in ``setup.py``: + +.. code-block:: python + + Extension(f"{PACKAGE_NAME}.Capsules.datetimetz", + sources=[ + 'src/cpy/Capsules/datetimetz.c', + 'src/cpy/Util/py_call_super.c', + ], + include_dirs=[ + '/usr/local/include', + 'src/cpy/Capsules', + 'src/cpy/Util', + ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +Extensive tests are in ``tests/unit/test_c_capsules.py``. + +-------------------------------- +Building +-------------------------------- + +Running ``python setup.py develop`` will build the extension(s) and then ``dtatetimetz`` can be imported: + +.. code-block:: python + + from cPyExtPatt.Capsules import datetimetz + + +-------------------------------- +Testing +-------------------------------- + +All the ests are in ``tests/unit/test_c_capsules.py``, but here is a relevant selection. +All depend on: + +.. code-block:: python + + import datetime + import zoneinfo + + import pytest + + from cPyExtPatt.Capsules import datetimetz + +A check on the ``__mro__``: + +.. code-block:: python + + def test_datetimetz_datetimetz_mro(): + mro = datetimetz.datetimetz.__mro__ + assert [str(v) for v in mro] == [ + "", + "", + "", + "", + ] + +A check on construction, first with a timezone, then without: + +.. code-block:: python + + @pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + (2024, 7, 15, 10, 21, 14), + {'tzinfo': zoneinfo.ZoneInfo('Europe/London')}, + '2024-07-15 10:21:14+01:00', + ), + ) + ) + def test_datetimetz_datetimetz_str(args, kwargs, expected): + d = datetimetz.datetimetz(*args, **kwargs) + assert str(d) == expected + + + @pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + (2024, 7, 15, 10, 21, 14), + {}, + 'No time zone provided.', + ), + ( + (2024, 7, 15, 10, 21, 14), + {'tzinfo': None, }, + 'No time zone provided.', + ), + ) + ) + def test_datetimetz_datetimetz_raises(args, kwargs, expected): + with pytest.raises(TypeError) as err: + d = datetimetz.datetimetz(*args, **kwargs) + assert err.value.args[0] == expected + + +Check the ``repr()``. +Note how this uses inheritance correctly whilst getting the type right: + +.. code-block:: python + + + @pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + (2024, 7, 15, 10, 21, 14), + {'tzinfo': zoneinfo.ZoneInfo('Europe/London')}, + ( + "datetimetz.datetimetz(2024, 7, 15, 10, 21, 14," + " tzinfo=zoneinfo.ZoneInfo(key='Europe/London'))" + ), + ), + ) + ) + def test_datetimetz_datetimetz_repr(args, kwargs, expected): + d = datetimetz.datetimetz(*args, **kwargs) + assert repr(d) == expected + +Here is a test for setting the ``tzinfo`` directly. +The error is handled correctly by the superclass. + +.. code-block:: python + + def test_datetimetz_datetimetz_set_tzinfo_raises(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + with pytest.raises(AttributeError) as err: + d.tzinfo = None + assert err.value.args[0] == "attribute 'tzinfo' of 'datetime.datetime' objects is not writable" + +Check that ``.replace()`` works as expected with ``tzinfo``: + +.. code-block:: python + + def test_datetimetz_datetimetz_replace_raises_tzinfo(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + with pytest.raises(TypeError) as err: + d.replace(tzinfo=None) + assert err.value.args[0] == 'No time zone provided.' + +Some equality tests. +We want to fail when comparing our ``datetimetz`` with a naive ``datatime`` object. + +.. code-block:: python + + def test_datetimetz_datetimetz_equal(): + d_tz = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London')) + d = datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + assert d_tz == d + + + def test_datetime_datetime_equal_naive(): + d = datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + d_no_tz = datetime.datetime(2024, 7, 15, 10, 21, 14) + assert d_no_tz != d + +Some datetime subtraction tests that show our ``datetimetz`` inter-operates correctly with itself and a ``datetime`` +object. + +.. code-block:: python + + @pytest.mark.parametrize( + 'd_tz, d, expected', + ( + ( + datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ), + datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ), + datetime.timedelta(0), + ), + ( + datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ), + datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('America/New_York') + ), + datetime.timedelta(seconds=-5 * 60 * 60), + ), + ( + datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('America/New_York') + ), + datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ), + datetime.timedelta(seconds=5 * 60 * 60), + ), + ) + ) + def test_datetimetz_datetimetz_subtract(d_tz, d, expected): + assert (d_tz - d) == expected + + + @pytest.mark.parametrize( + 'd_tz, d, expected', + ( + ( + datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ), + datetime.datetime(2024, 7, 15, 10, 21, 14), + '', + ), + ) + ) + def test_datetimetz_datetimetz_subtract_raises(d_tz, d, expected): + with pytest.raises(TypeError) as err: + d_tz - d + assert err.value.args[0] == "can't subtract offset-naive and offset-aware datetimes" + diff --git a/doc/sphinx/source/code_layout.rst b/doc/sphinx/source/code_layout.rst index 26f7d39..46c6e29 100644 --- a/doc/sphinx/source/code_layout.rst +++ b/doc/sphinx/source/code_layout.rst @@ -4,38 +4,59 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Source Code Layout + ================================= Source Code Layout ================================= I find it useful to physically separate out the source code into different categories: -+-------------------+------------+--------------------------+-----------+----------+--------------------------------------------------+ -| Category | Language | ``#include ``? | Testable? | Where? | Description | -+===================+============+==========================+===========+==========+==================================================+ -| Pure Python | Python | No | Yes | ``py/`` | Regular Python code tested by pytest or similar. | -+-------------------+------------+--------------------------+-----------+----------+--------------------------------------------------+ -| CPython interface | Mostly C | Yes | No | ``cpy/`` | C code that defines Python modules and classes. | -| | | | | | Functions that are exposed directly to Python. | -+-------------------+------------+--------------------------+-----------+----------+--------------------------------------------------+ -| CPython utilities | C, C++ | Yes | Yes | ``cpy/`` | Utility C/C++ code that works with Python | -| | | | | | objects but these functions that are *not* | -| | | | | | exposed directly to Python. | -| | | | | | This code can be tested in a C/C++ environment | -| | | | | | with a specialised test framework. | -| | | | | | See :ref:`cpp_and_cpython` for some examples. | -+-------------------+------------+--------------------------+-----------+----------+--------------------------------------------------+ -| C/C++ core | C, C++ | No | Yes | ``cpp/`` | C/C++ code that knows nothing about Python. | -| | | | | | This code can be tested in a C/C++ environment | -| | | | | | with a standard C/C++ test framework. | -+-------------------+------------+--------------------------+-----------+----------+--------------------------------------------------+ - +.. list-table:: Recommended Code Directories + :widths: 10 10 10 10 10 30 + :header-rows: 1 + + * - Category + - Language + - ``#include ``? + - Testable? + - Where? + - Description + * - Pure Python + - Python + - No + - Yes + - ``py/`` + - Regular Python code tested by pytest or similar. + * - CPython interface + - Mostly C + - Yes + - No + - ``cpy/`` + - C code that defines Python modules and classes. Functions that are exposed directly to Python. + * - CPython utilities + - C, C++ + - Yes + - Yes + - ``cpy/`` + - Utility C/C++ code that works with Python objects but these functions that are *not* exposed directly to Python. + This code can be tested in a C/C++ environment with a specialised test framework. + See :ref:`cpp_and_cpython` for some examples. + * - C/C++ core + - C, C++ + - No + - Yes + - ``cpp/`` + - C/C++ code that knows nothing about Python. This code can be tested in a C/C++ environment with a standard C/C++ + test framework. -------------------------------------- Testing CPython Utility Code -------------------------------------- -When making Python C API calls from a C/C++ environment it is important to initialise the Python interpreter. For example, this small program segfaults: +When making Python C API calls from a C/C++ environment it is important to initialise the Python interpreter. +For example, this small program segfaults: .. code-block:: c :linenos: @@ -51,13 +72,15 @@ When making Python C API calls from a C/C++ environment it is important to initi return 0; } -The reason is that ``PyErr_Format`` calls ``PyThreadState *thread_state = PyThreadState_Get();`` theen ``thread_state`` will be NULL unless the Python interpreter is initialised. +The reason is that ``PyErr_Format`` calls ``PyThreadState *thread_state = PyThreadState_Get();`` then ``thread_state`` +will be NULL unless the Python interpreter is initialised. -So you need to call ``Py_Initialize()`` to set up statically allocated interpreter data. Alternativley put ``if (! Py_IsInitialized()) Py_Initialize();`` in every test. See: `https://docs.python.org/3/c-api/init.html `_ +So you need to call ``Py_Initialize()`` to set up statically allocated interpreter data. +Alternatively put ``if (! Py_IsInitialized()) Py_Initialize();`` in every test. See: `https://docs.python.org/3/c-api/init.html `_ -Here are a couple of useful C++ functions that assert all is well that can be used at the begining of any function: +Here are a couple of useful C++ functions that assert all is well that can be used at the beginning of any function: -.. code-block:: cpp +.. code-block:: c /* Returns non zero if Python is initialised and there is no Python error set. * The second version also checks that the given pointer is non-NULL diff --git a/doc/sphinx/source/compiler_flags.rst b/doc/sphinx/source/compiler_flags.rst index e7e7fd7..7888242 100644 --- a/doc/sphinx/source/compiler_flags.rst +++ b/doc/sphinx/source/compiler_flags.rst @@ -4,36 +4,45 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Compiler Flags + ================================= Setting Compiler Flags ================================= -It is sometimes difficult to decide what flags to set for the compiler and the best advice is to use the same flags that the version of Python you are using was compiled with. Here are a couple of ways to do that. +It is sometimes difficult to decide what flags to set for the compiler and the best advice is to use the same flags that +the version of Python you are using was compiled with. Here are a couple of ways to do that. +.. index:: + single: Compiler Flags; CLI --------------------------------- From the Command Line --------------------------------- -In the Python install directory there is a `pythonX.Y-config` executable that can be used to extract the compiler flags where X is the major version and Y the minor version. For example (output is wrapped here for clarity): +In the Python install directory there is a `pythonX.Y-config` executable that can be used to extract the compiler flags +where X is the major version and Y the minor version. For example (output is wrapped here for clarity): .. code-block:: sh - $ which python - /usr/bin/python - $ python -V - Python 2.7.5 - $ /usr/bin/python2.7-config --cflags - -I/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7 - -I/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7 - -fno-strict-aliasing -fno-common -dynamic -arch x86_64 -arch i386 -g -Os -pipe - -fno-common -fno-strict-aliasing -fwrapv -DENABLE_DTRACE -DMACOSX -DNDEBUG -Wall - -Wstrict-prototypes -Wshorten-64-to-32 -DNDEBUG -g -fwrapv -Os -Wall - -Wstrict-prototypes -DENABLE_DTRACE + $ which python3 + /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 + $ python3 -VV + Python 3.13.0b3 (v3.13.0b3:7b413952e8, Jun 27 2024, 09:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ /Library/Frameworks/Python.framework/Versions/3.13/bin/python3-config --cflags + -I/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13 + -I/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13 + -fno-strict-overflow -Wsign-compare -Wunreachable-code -fno-common -dynamic -DNDEBUG + -g -O3 -Wall -arch arm64 -arch x86_64 -g + +.. index:: + single: Compiler Flags; Programmatically + single: sysconfig; importing -------------------------------------------------- -Programatically from Within a Python Process +Programmatically from Within a Python Process -------------------------------------------------- The ``sysconfig`` module contains information about the build environment for the particular version of Python: @@ -42,19 +51,23 @@ The ``sysconfig`` module contains information about the build environment for th >>> import sysconfig >>> sysconfig.get_config_var('CFLAGS') - '-fno-strict-aliasing -fno-common -dynamic -arch x86_64 -arch i386 -g -Os -pipe -fno-common -fno-strict-aliasing -fwrapv -DENABLE_DTRACE -DMACOSX -DNDEBUG -Wall -Wstrict-prototypes -Wshorten-64-to-32 -DNDEBUG -g -fwrapv -Os -Wall -Wstrict-prototypes -DENABLE_DTRACE' + '-fno-strict-overflow -Wsign-compare -Wunreachable-code -fno-common -dynamic -DNDEBUG -g -O3 -Wall -arch arm64 -arch x86_64 -g' >>> import pprint >>> pprint.pprint(sysconfig.get_paths()) - {'data': '/System/Library/Frameworks/Python.framework/Versions/2.7', - 'include': '/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7', - 'platinclude': '/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7', - 'platlib': '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages', - 'platstdlib': '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7', - 'purelib': '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages', - 'scripts': '/System/Library/Frameworks/Python.framework/Versions/2.7/bin', - 'stdlib': '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7'} + {'data': '/Library/Frameworks/Python.framework/Versions/3.13', + 'include': '/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13', + 'platinclude': '/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13', + 'platlib': '/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages', + 'platstdlib': '/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13', + 'purelib': '/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages', + 'scripts': '/Library/Frameworks/Python.framework/Versions/3.13/bin', + 'stdlib': '/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13'} >>> sysconfig.get_paths()['include'] - '/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7' + '/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13' + +.. index:: + single: Compiler Flags; sysconfig + single: sysconfig; CLI -------------------------------------------------- From the Command Line using ``sysconfig`` @@ -65,49 +78,43 @@ This very verbose output will give you a complete picture of your environment: .. code-block:: sh $ python3 -m sysconfig - Platform: "macosx-10.6-intel" - Python version: "3.4" + Platform: "macosx-10.13-universal2" + Python version: "3.13" Current installation scheme: "posix_prefix" - Paths: - data = "/Library/Frameworks/Python.framework/Versions/3.4" - include = "/Library/Frameworks/Python.framework/Versions/3.4/include/python3.4m" - platinclude = "/Library/Frameworks/Python.framework/Versions/3.4/include/python3.4m" - platlib = "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages" - platstdlib = "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4" - purelib = "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages" - scripts = "/Library/Frameworks/Python.framework/Versions/3.4/bin" - stdlib = "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4" - - Variables: - ABIFLAGS = "m" - AC_APPLE_UNIVERSAL_BUILD = "1" + Paths: + data = "/Library/Frameworks/Python.framework/Versions/3.13" + include = "/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13" + platinclude = "/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13" + platlib = "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages" + platstdlib = "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13" + purelib = "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages" + scripts = "/Library/Frameworks/Python.framework/Versions/3.13/bin" + stdlib = "/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13" + + Variables: + ABIFLAGS = "" + AC_APPLE_UNIVERSAL_BUILD = "0" + AIX_BUILDDATE = "0" AIX_GENUINE_CPLUSPLUS = "0" - AR = "ar" - ARFLAGS = "rc" - ASDLGEN = "python /Users/sysadmin/build/v3.4.4/Parser/asdl_c.py" - ASDLGEN_FILES = "/Users/sysadmin/build/v3.4.4/Parser/asdl.py /Users/sysadmin/build/v3.4.4/Parser/asdl_c.py" - AST_ASDL = "/Users/sysadmin/build/v3.4.4/Parser/Python.asdl" - AST_C = "Python/Python-ast.c" - AST_C_DIR = "Python" - AST_H = "Include/Python-ast.h" - AST_H_DIR = "Include" - BASECFLAGS = "-fno-strict-aliasing -fno-common -dynamic" - BASECPPFLAGS = "" - BASEMODLIBS = "" - BINDIR = "/Library/Frameworks/Python.framework/Versions/3.4/bin" - BINLIBDEST = "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4" - ... + ALIGNOF_LONG = "8" + ALIGNOF_MAX_ALIGN_T = "8" + ALIGNOF_SIZE_T = "8" + ALT_SOABI = "0" + ANDROID_API_LEVEL = "0" + AR = "/usr/bin/xcrun ar" + ... +.. index:: + single: Compiler Flags; setup.py -------------------------------------------------- Setting Flags Automatically in ``setup.py`` -------------------------------------------------- -The sysconfig module allows you to create a generic ``setup.py`` script for Python C extensions (see highlighted line): +The sysconfig module allows you to create a generic ``setup.py`` script for Python C extensions, something along these lines: .. code-block:: python - :emphasize-lines: 15 from distutils.core import setup, Extension import os @@ -129,7 +136,7 @@ The sysconfig module allows you to create a generic ``setup.py`` script for Pyth extra_compile_args += ["-g3", "-O0", "-DDEBUG=%s" % _DEBUG_LEVEL, "-UNDEBUG"] else: extra_compile_args += ["-DNDEBUG", "-O3"] - + setup( name = '...', version = '...', @@ -154,6 +161,7 @@ The sysconfig module allows you to create a generic ``setup.py`` script for Pyth '.', '...', os.path.join(os.getcwd(), 'include'), + sysconfig.get_paths()['include'], ], library_dirs = [os.getcwd(),], # path to .a or .so file(s) extra_compile_args=extra_compile_args, @@ -161,3 +169,73 @@ The sysconfig module allows you to create a generic ``setup.py`` script for Pyth ), ] ) + +.. index:: + pair: Compiler Flags; CMake + +---------------------------------------- +Getting C/C++ Flags from a CMake Build +---------------------------------------- + +If your project can be built under CMake (such as this one can be) then the compiler flags can be obtained by looking +at the generated file ``cmake-build-debug/CMakeFiles/.dir/flags.make`` (debug) or +``cmake-build-release/CMakeFiles/.dir/flags.make`` (release). + +For example, for this project the file is ``cmake-build-debug/CMakeFiles/PythonExtensionPatterns.dir/flags.make``. + +This looks something like this (wrapped for clarity and replaced user with ) : + +.. code-block:: text + + # CMAKE generated file: DO NOT EDIT! + # Generated by "Unix Makefiles" Generator, CMake Version 3.28 + + # compile C with \ + # /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc + # compile CXX with \ + # /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++ + C_DEFINES = + + C_INCLUDES = -I/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13 \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/cpy \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/cpy/Containers \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/cpy/Watchers + + C_FLAGSarm64 = -g -std=gnu11 -arch arm64 -isysroot \ + /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.3.sdk \ + -fcolor-diagnostics -Wall -Wextra -Wpedantic -Werror -Wfatal-errors \ + -Wno-unused-variable -Wno-unused-parameter -fexceptions \ + -Wno-c99-extensions -Wno-c++11-extensions -O0 -g3 -ggdb \ + -Wno-unused-function + + C_FLAGS = -g -std=gnu11 -arch arm64 -isysroot \ + /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.3.sdk \ + -fcolor-diagnostics -Wall -Wextra -Wpedantic -Werror -Wfatal-errors \ + -Wno-unused-variable -Wno-unused-parameter -fexceptions \ + -Wno-c99-extensions -Wno-c++11-extensions -O0 -g3 -ggdb \ + -Wno-unused-function + + CXX_DEFINES = + + CXX_INCLUDES = -I/Library/Frameworks/Python.framework/Versions/3.13/include/python3.13 \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/cpy \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/cpy/Containers \ + -I/Users//CLionProjects/PythonExtensionPatterns/src/cpy/Watchers + + CXX_FLAGSarm64 = -g -std=gnu++17 -arch arm64 -isysroot \ + /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.3.sdk \ + -fcolor-diagnostics -Wall -Wextra -Wpedantic -Werror -Wfatal-errors \ + -Wno-unused-variable -Wno-unused-parameter -fexceptions \ + -Wno-c99-extensions -Wno-c++11-extensions -O0 -g3 -ggdb \ + -Wno-unused-function + + CXX_FLAGS = -g -std=gnu++17 -arch arm64 -isysroot \ + /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.3.sdk \ + -fcolor-diagnostics -Wall -Wextra -Wpedantic -Werror -Wfatal-errors \ + -Wno-unused-variable -Wno-unused-parameter -fexceptions \ + -Wno-c99-extensions -Wno-c++11-extensions -O0 -g3 -ggdb \ + -Wno-unused-function + +This would be fairly easy to parse, perhaps by ``setup.py``. diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 020ae23..64f7103 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -47,16 +47,20 @@ # General information about the project. project = u'Python Extension Patterns' -copyright = u'2014, Paul Ross' +copyright = u'2014-2025, Paul Ross All rights reserved.' +author = 'Paul Ross ' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.1.0' +version = '0.3' # The full version, including alpha/beta/rc tags. -release = '0.1.0' +release = '0.3.0' + +todo_include_todos = True +todo_link_only = True # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -193,14 +197,17 @@ # Additional stuff for the LaTeX preamble. #'preamble': '', + 'preamble': r'''\usepackage{lscape}''', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'PythonExtensionPatterns.tex', u'Python Extension Patterns Documentation', - u'Paul Ross', 'manual'), + ( + 'index', 'PythonExtensionPatterns.tex', u'Python Extension Patterns', + 'Paul Ross ', 'manual', + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -229,7 +236,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'pythonextensionpatterns', u'Python Extension Patterns Documentation', + ('index', 'pythonextensionpatterns', u'Python Extension Patterns', [u'Paul Ross'], 1) ] @@ -243,7 +250,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'PythonExtensionPatterns', u'Python Extension Patterns Documentation', + ('index', 'PythonExtensionPatterns', u'Python Extension Patterns', u'Paul Ross', 'PythonExtensionPatterns', 'One line description of project.', 'Miscellaneous'), ] diff --git a/doc/sphinx/source/containers_and_refcounts.rst b/doc/sphinx/source/containers_and_refcounts.rst new file mode 100644 index 0000000..5703b14 --- /dev/null +++ b/doc/sphinx/source/containers_and_refcounts.rst @@ -0,0 +1,1969 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + +.. highlight:: python + :linenothreshold: 20 + +.. toctree:: + :maxdepth: 3 + +.. + Links, mostly to the Python documentation. + Specific container links are just before the appropriate section. + +.. _Py_BuildValue(): https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue + + +.. index:: + pair: Documentation Lacunae; Containers + +.. _chapter_containers_and_refcounts: + +====================================== +Containers and Reference Counts +====================================== + +This chapter looks in more detail of how the Python C API works with different containers, +such as ``tuple``, ``list``, ``set`` and ``dict`` [#]_. +It also clarifies and correct the Python documentation where that is inaccurate, incomplete or misleading. +This also shows where the Python C API has some undocumented failure modes, some of which can lead to undefined +behaviour. + +This chapter includes examples and tests that you can step through to better understand the interplay +between the container and the objects in that container. + +Of particular interest are *Setters*, *Getters* and the behaviour of ``Py_BuildValue`` for each of those +containers [#]_. + + +.. note:: + + The Python documentation for the + `Concrete Objects Layer `_ + has a general warning that passing ``NULL`` values into functions gives rise to undefined behaviour. + This is not always the case and this chapter explores this in more detail than the official Python documentation. + +--------------------------- +Some Additional Terminology +--------------------------- + +As well as :ref:`chapter_refcount.new`, :ref:`chapter_refcount.stolen` and :ref:`chapter_refcount.borrowed` +described in the previous chapter some other *behaviors* of containers are worth defining when they interact +with objects. + +.. index:: + single: Reference Counts; Discarded + +.. _chapter_containers_and_refcounts.discarded: + +Discarded References +--------------------------- + +This is when a container has a reference to an object but is required to replace it with another object. +In this case the container decrements the reference count of the original object before replacing it with the other +object. +This is to prevent a memory leak of the previous object. + +.. warning:: + + If the the replacement object is the **same** as the existing object then very bad things *might* happen. + For example see the warning in `PyTuple_SetItem()`_ + :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem.replacement`. + +.. index:: + single: Reference Counts; Abandoned + +.. _chapter_containers_and_refcounts.abandoned: + +Abandoned References +--------------------------- + +This is when a container has a reference to an object but is required to replace it with another object. +In this case the container *does not* decrement the reference count of the original object before replacing it with +the other object. +This *will* lead to a memory leak *unless* the replacement object is the same as the existing object. +Of course if the original reference is ``NULL`` there is no leak. + +An example of this is ``PyTuple_SET_ITEM()`` +:ref:`chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM.replacement`. + +--------------------------- +Exploring the CPython C API +--------------------------- + +The code in this chapter explores the CPython C API in several ways: + +* C code that can be stepped through in the debugger. + This code is in ``src/cpy/Containers/DebugContainers.c``. + This uses ``asserts`` to check the results, particularly reference counts, so should always be compiled with + ``-DDEBUG``. + The tests are exercised by ``src/main.c``. +* Test code is in ``src/cpy/RefCount/cRefCount.c`` which is built into the Python module + ``cPyExtPatt.cRefCount``. + This can be run under ``pytest`` for multiple Python versions by ``build_all.sh``. +* A study of the Python source code. +* A review of the Python C API documentation. + +.. note:: + + The examples below use code that calls a function ``new_unique_string()``. + This function is designed to create a new, unique, ``PyObject`` (a string) + that is never cached so always starts with a reference count of unity. + The implementation is in ``src/cpy/Containers/DebugContainers.c`` and looks something like this: + + .. code-block:: c + + static long debug_test_count = 0L; + + PyObject * + new_unique_string(const char *function, const char *suffix) { + if (suffix){ + return PyUnicode_FromFormat( + "%s-%s-%ld", function, suffix, debug_test_count++ + ); + } + return PyUnicode_FromFormat("%s-%ld", function, debug_test_count++); + } + +Here is an example test function that checks that `PyTuple_SetItem()`_ *steals* a reference: + +.. code-block:: c + + /** + * A function that checks whether a tuple steals a reference when using PyTuple_SetItem. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ + void dbg_PyTuple_SetItem_steals(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyTuple_SetItem(container, 0, value)) { + assert(0); + } + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + assert(!PyErr_Occurred()); + } + +Firstly Tuples, I'll go into quite a lot of detail here because it is very similar to the +C API for lists which I'll cover with more brevity in a later section. + +.. _chapter_containers_and_refcounts.tuples: + +.. + Links, mostly to the Python documentation: + +.. _PyTuple_SetItem(): https://docs.python.org/3/c-api/tuple.html#c.PyTuple_SetItem +.. _PyTuple_SET_ITEM(): https://docs.python.org/3/c-api/tuple.html#c.PyTuple_SET_ITEM +.. _PyTuple_Pack(): https://docs.python.org/3/c-api/tuple.html#c.PyTuple_Pack +.. _PyTuple_GetItem(): https://docs.python.org/3/c-api/tuple.html#c.PyTuple_GetItem +.. _PyTuple_GET_ITEM(): https://docs.python.org/3/c-api/tuple.html#c.PyTuple_GET_ITEM + + +.. index:: + single: Tuple + +----------------------- +Tuples +----------------------- + +The Python documentation for the `Tuple C API `_ +is here. + +Firstly setters, there are two APIs for setting an item in a tuple; `PyTuple_SetItem()`_ and `PyTuple_SET_ITEM()`_. + +.. _chapter_containers_and_refcounts.tuples.PyTuple_SetItem: + +.. index:: + single: PyTuple_SetItem() + single: Tuple; PyTuple_SetItem() + +``PyTuple_SetItem()`` +--------------------- + +`PyTuple_SetItem()`_ (a C function) inserts an object into a tuple with error checking. +This function returns non-zero on error, these are described below in +:ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem.failures`. +The failure of `PyTuple_SetItem()`_ has serious consequences for the value +that is intended to be inserted. + +``PyTuple_SetItem()`` Basic Usage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyTuple_SetItem()`_ *steals* a reference, that is, the container assumes responsibility +for freeing that value when the container is free'd ('freeing' meaning decrementing the reference count). +For example: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); /* container ref count will be 1. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); /* value ref count will be 1. */ + PyTuple_SetItem(container, 0, value); /* value ref count will remain at 1. */ + /* get_item == value and value ref count will be 1. */ + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + /* The contents of the container and value, will be decref'd + * In this particular case both will go to zero and free'd. */ + Py_DECREF(container); + /* Do not do this as the container has dealt with this. */ + /* Py_DECREF(value); */ + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SetItem_steals`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SetItem_steals()`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyTuple_SetItem_steals()``. + +Whilst we are here this is an example of testing the behaviour by manipulating reference counts which we then check +with ``assert()``. +The rationale is that you can't check reference counts after an object is destroyed. +For example: + +.. code-block:: c + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + assert(Py_REFCNT(value) == 1); + Py_DECREF(value); + /* This will fail, the reference count will have an arbitrary value. */ + assert(Py_REFCNT(value) == 0); + +Once an object has been free'd you can not rely on the reference count field. +Instead, deliberately increment the reference count before the critical section and check it afterwards. +For example: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + assert(Py_REFCNT(value) == 1); + PyTuple_SetItem(container, 0, value); + assert(Py_REFCNT(value) == 1); + /* Increment the reference count so we can see destruction. */ + Py_INCREF(value); + assert(Py_REFCNT(value) == 2); + /* Check destruction. */ + Py_DECREF(container); + assert(Py_REFCNT(value) == 1); + /* Clean up. */ + Py_DECREF(value); + +.. index:: + single: PyTuple_SetItem(); Replacement + single: Documentation Lacunae; PyTuple_SetItem() Replacement + +.. _chapter_containers_and_refcounts.tuples.PyTuple_SetItem.replacement: + +``PyTuple_SetItem()`` Replacement +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +What happens when you use `PyTuple_SetItem()`_ to replace an existing element in a tuple? +`PyTuple_SetItem()`_ still *steals* a reference, but what happens to the original reference? +Lets see: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); /* container ref count will be 1. */ + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + /* value_a ref count will be 1. */ + PyTuple_SetItem(container, 0, value_a); /* value_a ref count will remain 1. */ + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + /* value_b ref count will be 1. */ + PyTuple_SetItem(container, 0, value_b); + /* Now value_b ref count will remain 1 and value_a ref count will have been decremented + * In this case value_a will have been free'd. */ + +.. warning:: + + What happens if you use `PyTuple_SetItem()`_ to replace a value with the *same* value? + For example: + + .. code-block:: c + + PyObject *container = PyTuple_New(1); + /* container ref count will be 1. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* value ref count will be 1. */ + PyTuple_SetItem(container, 0, value); + /* value ref count is still 1 as it has been *stolen*. */ + + Now repeat the replacement: + + .. code-block:: c + + /* Repeating the same call will only lead to trouble later on (maybe). */ + PyTuple_SetItem(container, 0, value); + /* And this will segfault as, during execution, it will + * try to decrement a value that does not exist. */ + Py_DECREF(container); + /* So what is going on? */ + + What is happening is that the second time `PyTuple_SetItem()`_ is called it decrements the reference count of the + existing member that happens to be ``value``. + In this case this brings ``value``'s reference count from one down to zero + At that point ``value`` is free'd. + Then `PyTuple_SetItem()`_ blithely sets ``value`` which is now, likely, garbage. + + This is not described in the Python documentation. + + A simple change to `PyTuple_SetItem()`_ would prevent this from producing undefined behaviour by checking if the + replacement is the same as the existing value. + + `PyTuple_SET_ITEM()`_ does not exhibit this problem as it *abandons* values rather than *discarding* them. + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SetItem_steals_replace`` + * ``dbg_PyTuple_SetItem_replace_with_same`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SetItem_steals_replace`` + * ``test_PyTuple_SetItem_replace_same`` +* Python: in ``tests.unit.test_c_ref_count`` + * ``test_PyTuple_SetItem_steals_replace()`` + * ``test_PyTuple_SetItem_replace_same()`` + +.. index:: + single: PyTuple_SetItem(); Failures + pair: Documentation Lacunae; PyTuple_SetItem() Failures + +.. _chapter_containers_and_refcounts.tuples.PyTuple_SetItem.failures: + +``PyTuple_SetItem()`` Failures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyTuple_SetItem()`_ can return a non-zero error code for these reasons: + +* The given container is not a tuple. +* The index is out of range; index < 0 or index >= tuple length (negative indexes are not allowed). + +.. warning:: + + A consequence of failure is that the value being inserted will be decref'd. + For example this code will segfault: + + .. code-block:: c + + PyObject *container = PyTuple_New(1); /* Reference count will be 1. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); /* Ref count will be 1. */ + PyTuple_SetItem(container, 1, value); /* Index out of range. */ + Py_DECREF(value); /* value has already been decref'd and free'd so this will SIGSEGV */ + + This is not described in the Python documentation. + +For code tests see, when the container is not a tuple: + +* C: ``dbg_PyTuple_SetItem_fails_not_a_tuple`` in ``src/cpy/Containers/DebugContainers.c``. +* CPython: ``test_PyTuple_SetItem_fails_not_a_tuple`` in ``src/cpy/RefCount/cRefCount.c``. +* Python: ``tests.unit.test_c_ref_count.test_PyTuple_SetItem_fails_not_a_tuple``. + +And, when the index out of range: + +* C: ``dbg_PyTuple_SetItem_fails_out_of_range`` in ``src/cpy/Containers/DebugContainers.c``. +* CPython: ``test_PyTuple_SetItem_fails_out_of_range`` in ``src/cpy/RefCount/cRefCount.c``. +* Python: ``tests.unit.test_c_ref_count.test_PyTuple_SetItem_fails_out_of_range``. + +.. note:: + + I'm not really sure why the `PyTuple_SetItem()`_ API exists. + Tuples are meant to be immutable but this API treats existing tuples as mutable. + It would seem like `PyTuple_SET_ITEM()`_ would be enough. + +.. _chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM: + +.. index:: + single: PyTuple_SET_ITEM() + single: Tuple; PyTuple_SET_ITEM(); + pair: Documentation Lacunae; PyTuple_SET_ITEM() Replacement + +``PyTuple_SET_ITEM()`` +---------------------- + +`PyTuple_SET_ITEM()`_ is a function like macro that inserts an object into a tuple without any error checking +(see :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM.failures` below). +However type checking is performed as an assertion if Python is built in +`debug mode `_ or +`with assertions `_. +Because of the absence of checks, it is slightly faster than `PyTuple_SetItem()`_ . +This is usually used on newly created tuples. + +Importantly `PyTuple_SET_ITEM()`_ behaves **differently** to `PyTuple_SetItem()`_ +when replacing another object. + +``PyTuple_SET_ITEM()`` Basic Usage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyTuple_SET_ITEM()`_ *steals* a reference just like `PyTuple_SetItem()`_. + +.. code-block:: c + + PyObject *container = PyTuple_New(1); /* Reference count will be 1. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); /* Ref count will be 1. */ + PyTuple_SET_ITEM(container, 0, value); /* Ref count of value will be 1. */ + /* get_item == value and Ref count will be 1. */ + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + assert(get_item == value && Py_REFCNT(value) == 1); + Py_DECREF(container); /* The contents of the container, value, will be decref'd */ + /* Do not do this as the container deals with this. */ + /* Py_DECREF(value); */ + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_PyTuple_SET_ITEM_steals()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_PyTuple_SET_ITEM_steals()`` +* Python: in ``tests.unit.test_c_ref_count`` + * ``test_PyTuple_PyTuple_SET_ITEM_steals()`` + +.. index:: + single: PyTuple_SET_ITEM(); Replacement + +.. _chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM.replacement: + +``PyTuple_SET_ITEM()`` Replacement +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyTuple_SET_ITEM()`_ **differs** from `PyTuple_SetItem()`_ when replacing an existing +element in a tuple as the original reference will be leaked. +This is because `PyTuple_SET_ITEM()`_ *abandons* the previous reference +(see :ref:`chapter_containers_and_refcounts.abandoned`): + +.. code-block:: c + + PyObject *container = PyTuple_New(1); /* Reference count will be 1. */ + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); /* Ref count will be 1. */ + PyTuple_SET_ITEM(container, 0, value_a); /* Ref count of value_a will be 1. */ + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); /* Ref count will be 1. */ + PyTuple_SET_ITEM(container, 0, value_b); + assert(Py_REFCNT(value_a) == 1); + /* Ref count of value_b will be 1, + * value_a ref count will still be at 1 and value_a will be leaked unless decref'd. */ + +.. note:: + + Because `PyTuple_SET_ITEM()`_ *abandons* the previous reference it does not have the problem with + undefined behaviour that `PyTuple_Set_Item()`_ has. + For that see the warning about undefined behaviour in `PyTuple_Set_Item()`_ + :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem.replacement`. + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SET_ITEM_steals_replace()`` + * ``dbg_PyTuple_SET_ITEM_replace_with_same()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SetItem_steals_replace()`` + * ``test_PyTuple_SET_ITEM_replace_same()`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyTuple_SetItem_steals_replace`` + * ``test_PyTuple_SET_ITEM_replace_same``. + +.. index:: + single: PyTuple_SET_ITEM(); Failures + +.. _chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM.failures: + +.. index:: + pair: Documentation Lacunae; PyTuple_SET_ITEM() Failures + +``PyTuple_SET_ITEM()`` Failures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyTuple_SET_ITEM()`_ minimises the error checking that `PyTuple_SetItem()`_ does, so: + +* It does not check if the given container is a Tuple. +* It does not check if the index in range. + +If either of those is wrong then `PyTuple_SET_ITEM()`_ is capable of writing to arbitrary +memory locations, and the result is likely to be tragic, mostly undefined behaviour +and/or memory corruption. + +Setting and Replacing ``NULL`` +------------------------------ + +This looks at what happens when setting or replacing a ``NULL`` pointer in a tuple. +Both `PyTuple_SetItem()`_ and `PyTuple_SET_ITEM()`_ behave the same way. + +.. index:: + single: PyTuple_SetItem(); Setting NULL + single: PyTuple_SET_ITEM(); Setting NULL + pair: Documentation Lacunae; PyTuple Setting NULL + +Setting ``NULL`` +^^^^^^^^^^^^^^^^ + +Setting a ``NULL`` will not cause an error: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); + assert(!PyErr_Occurred()); + PyTuple_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SetIem_NULL()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SetItem_NULL()`` +* Python: in ``tests/unit/test_c_ref_count.py`` + * ``test_PyTuple_SetItem_NULL()``. + +And: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); + assert(!PyErr_Occurred()); + PyTuple_SET_ITEM(container, 0, NULL); + assert(!PyErr_Occurred()); + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SET_ITEM_NULL()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SET_ITEM_NULL()`` +* Python: in ``tests/unit/test_c_ref_count.py`` + * ``test_PyTuple_SET_ITEM_NULL()``. + +.. index:: + single: PyTuple_SetItem(); Replacing NULL + single: PyTuple_SET_ITEM(); Replacing NULL + pair: Documentation Lacunae; PyTuple Replacing NULL + +Replacing ``NULL`` +^^^^^^^^^^^^^^^^^^ + +Replacing a ``NULL`` will not cause an error, the original value is *abandoned* +(see :ref:`chapter_containers_and_refcounts.abandoned`) and the replaced value reference is *stolen* +(see :ref:`chapter_refcount.stolen`): + +.. code-block:: c + + PyObject *container = PyTuple_New(1); + PyTuple_SetItem(container, 0, NULL); + PyObject *value = new_unique_string(__FUNCTION__, NULL); /* Ref Count of value is 1. */ + PyTuple_SetItem(container, 0, value); /* Ref Count of value is still 1. */ + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SetIem_NULL_SetIem`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SetItem_NULL_SetIem`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyTuple_SetItem_NULL_SetIem()``. + +And: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); + PyTuple_SET_ITEM(container, 0, NULL); + PyObject *value = new_unique_string(__FUNCTION__, NULL); /* Ref Count of value is 1. */ + PyTuple_SET_ITEM(container, 0, value); /* Ref Count of value is still 1. */ + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_SET_ITEM_NULL_SET_ITEM()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_SET_ITEM_NULL_SET_ITEM()`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyTuple_SET_ITEM_NULL_SET_ITEM()``. + +.. _chapter_containers_and_refcounts.tuples.PyTuple_Pack: + +.. index:: + single: PyTuple_Pack() + single: Tuple; PyTuple_Pack() + +``PyTuple_Pack()`` +------------------ + +`PyTuple_Pack()`_ takes a length and a variable argument list of PyObjects. +Each of those PyObjects reference counts will be incremented. +In that sense it behaves as `Py_BuildValue()`_. + +.. note:: + `PyTuple_Pack()`_ is implemented as a separate low level routine, it does not invoke + `PyTuple_SetItem()`_ or `PyTuple_SET_ITEM()`_ . + +For example: + +.. code-block:: c + + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + PyObject *container = PyTuple_Pack(2, value_a, value_b); + assert(Py_REFCNT(value_a) == 2); + assert(Py_REFCNT(value_b) == 2); + + Py_DECREF(container); + + /* Leaks: */ + assert(Py_REFCNT(value_a) == 1); + assert(Py_REFCNT(value_b) == 1); + /* Fix leaks: */ + Py_DECREF(value_a); + Py_DECREF(value_b); + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_PyTuple_Pack`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_Py_PyTuple_Pack`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyTuple_Py_PyTuple_Pack()``. + +.. _chapter_containers_and_refcounts.tuples.Py_BuildValue: + +.. index:: + single: Tuple; Py_BuildValue() + +``Py_BuildValue()`` +------------------- + +`Py_BuildValue()`_ is a very convenient way to create tuples, lists and dictionaries. +``Py_BuildValue("(O)", value);`` will increment the refcount of value and this can, +potentially, leak: + +.. code-block:: c + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* value reference count is 1. */ + PyObject *container = Py_BuildValue("(O)", value); + /* value reference count is incremented to 2. */ + assert(Py_REFCNT(value) == 2); + + Py_DECREF(container); + /* value reference count is decremented to 1. */ + assert(Py_REFCNT(value_a) == 1); + + /* value is leaked if Py_DECREF(value) is not called. */ + /* Fix leak. */ + Py_DECREF(value); + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyTuple_Py_BuildValue()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyTuple_Py_BuildValue`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyTuple_Py_BuildValue()``. + +.. index:: + single: PyTuple_GetItem() + single: Tuple; PyTuple_GetItem() + single: PyTuple_GET_ITEM() + single: Tuple; PyTuple_GET_ITEM() + pair: Getters; Tuple + +.. _chapter_containers_and_refcounts.tuples.Getters: + +.. index:: + single: Tuple; Getters + single: Tuple; PyTuple_GetItem() + single: Tuple; PyTuple_GET_ITEM() + +Tuple Getters +--------------------- + +There are these APIs for getting an item from a tuple: + +* `PyTuple_GetItem()`_, it returns a borrowed reference and will error + if the supplied container is not tuple or the index is negative or out of range. +* `PyTuple_GET_ITEM()`_, it returns a borrowed reference and there is + no error checking for the container being a tuple or the index being in range. + The type checking is performed as an assertion if Python is built in + `debug mode `_ or + `with assertions `_. + If not the results are undefined. + +.. index:: single: Tuple; API Summary + +Summary +---------------------- + +* `PyTuple_SetItem()`_ and `PyTuple_SET_ITEM()`_ *steal* references. +* `PyTuple_SetItem()`_ and `PyTuple_SET_ITEM()`_ behave differently when replacing an existing, different, value. +* `PyTuple_SetItem()`_ and `PyTuple_SET_ITEM()`_ behave differently when replacing the *same* value. + In particular `PyTuple_SetItem()`_ can produce undefined behaviour. +* If `PyTuple_SetItem()`_ errors it will decrement the reference count of the given value which can produce undefined + behaviour. +* `PyTuple_Pack()`_ and `Py_BuildValue()`_ increment reference counts and thus may leak. + +.. + Links, mostly to the Python documentation: + +.. _PyList_SetItem(): https://docs.python.org/3/c-api/list.html#c.PyList_SetItem +.. _PyList_SET_ITEM(): https://docs.python.org/3/c-api/list.html#c.PyList_SET_ITEM +.. _PyList_Append(): https://docs.python.org/3/c-api/list.html#c.PyList_Append +.. _PyList_Insert(): https://docs.python.org/3/c-api/list.html#c.PyList_Insert +.. _PyList_GetItem(): https://docs.python.org/3/c-api/list.html#c.PyList_GetItem +.. _PyList_GET_ITEM(): https://docs.python.org/3/c-api/list.html#c.PyList_GET_ITEM +.. _PyList_GetItemRef(): https://docs.python.org/3/c-api/list.html#c.PyList_GetItemRef + +.. index:: + single: List + pair: Documentation Lacunae; Lists + +.. _chapter_containers_and_refcounts.lists: + +----------------------- +Lists +----------------------- + +.. index:: + single: PyList_SetItem() + single: List; PyList_SetItem() + single: PyList_SET_ITEM() + single: List; PyList_SET_ITEM() + +.. _chapter_containers_and_refcounts.lists.PyList_SetItem: + +.. _chapter_containers_and_refcounts.lists.PyList_SET_ITEM: + +.. _chapter_containers_and_refcounts.lists.PyList_SET_ITEM.failures: + +.. index:: + pair: Documentation Lacunae; PyList_SetItem() + pair: Documentation Lacunae; PyList_SET_ITEM() + +``PyList_SetItem()`` and ``PyList_SET_ITEM()`` +---------------------------------------------- + +`PyList_SetItem()`_ and `PyList_SET_ITEM()`_ behave identically to their equivalents `PyTuple_SetItem()`_ +(link :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem`) +and `PyTuple_SET_ITEM()`_ (link :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM`). + +Note that, as with tuples, `PyList_SetItem()`_ and `PyList_SET_ITEM()`_ behave differently on replacement of values +(see :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM.replacement`). +The Python documentation on `PyList_SET_ITEM()`_ correctly identifies when a leak can occur +(unlike `PyTuple_SET_ITEM()`_). + +On replacement with `PyList_SetItem()`_ heed the warning in +:ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem.replacement` as `PyList_SetItem()`_ holds the same danger. + +`Py_BuildValue()`_ also behaves identically, as far as reference counts are concerned, with Lists as it does with +Tuples (see :ref:`chapter_containers_and_refcounts.tuples.Py_BuildValue`). + + +.. index:: + single: PyList_Append() + single: List; PyList_Append() + +.. _chapter_containers_and_refcounts.lists.PyList_Append: + +``PyList_Append()`` +--------------------- + +`PyList_Append()`_ (a C function) adds an object onto the end of a list with error checking. +This increments the reference count of the given value which will be decremented on container destruction. +For example: + +.. code-block:: c + + PyObject *container = PyList_New(0); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + PyList_Append(container, value); + assert(Py_REFCNT(value) == 2); + Py_DECREF(container); + /* A leak unless decref'd. */ + assert(Py_REFCNT(value) == 1); + +`PyList_Append()`_ uses `PyList_SET_ITEM()`_ in its implementation. +`PyList_Append()`_ can fail for two reasons: + +* The given container is not a list. +* The given value is NULL. + +On failure the reference count of value is unchanged and a ``SystemError`` is raised with the text +"bad argument to internal function". + +For code and tests, including failure modes, see: + +* C: ``dbg_PyList_Append...`` in ``src/cpy/Containers/DebugContainers.c``. +* CPython: ``test_PyList_Append...`` in ``src/cpy/RefCount/cRefCount.c``. +* Python: ``tests.unit.test_c_ref_count.test_PyList_Append`` etc. + +.. index:: + single: PyList_Insert() + single: List; PyList_Insert() + pair: Documentation Lacunae; PyList_Insert() + +.. _chapter_containers_and_refcounts.lists.PyList_Insert: + +``PyList_Insert()`` +--------------------- + +`PyList_Insert()`_ (a C function) inserts an object before a specific index in a list with error checking. +This increments the reference count of the given value which will be decremented on container destruction. + +`PyList_Insert()`_ is equivalent to ``list.insert(...)``, for example: + +.. code-block:: python + + >>> l = [] + >>> l.insert(123, "Hello") + >>> l.insert(-123, "World") + >>> l + ['World', 'Hello'] + +`PyList_Insert()`_ can fail for two reasons: + +* The given container is not a list. +* The given value is NULL. + +.. note:: + + `PyList_Insert()`_ does not fail because of any value of the index, either positive or negative (this is not + identified in the Python documentation). + +On failure the reference count of value is unchanged, `PyList_Insert()`_ returns -1, and a ``SystemError`` is set with +the text "bad argument to internal function". + +For code and tests, including failure modes, see: + +* C: ``dbg_PyList_Insert...`` in ``src/cpy/Containers/DebugContainers.c``. +* CPython: ``test_PyList_Insert...`` in ``src/cpy/RefCount/cRefCount.c``. +* Python: ``tests.unit.test_c_ref_count.test_PyList_Insert`` etc. + +.. + This note and code blocks are quite big in latex so page break here. Now commented out. + + .. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. note:: + + The Python documentation does not mention (but implies) that if the index is greater than the list length then the + value is appended to the list. + + For example (``dbg_PyList_Insert_Is_Truncated()`` in ``src/cpy/Containers/DebugContainers.c``): + + .. code-block:: c + + PyObject *container = PyList_New(0); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + // Insert before index 4 + if (PyList_Insert(container, 4L, value)) { + assert(0); + } + assert(Py_REFCNT(value) == 2); + PyObject *get_item; + // PyList_Insert at 4 actually inserts at 0. + assert(PyList_GET_SIZE(container) == 1L); + get_item = PyList_GET_ITEM(container, 0L); + assert(get_item == value); + +.. note:: + + Neither the Python documentation for `PyList_Insert()`_ or ``list.insert()`` does not make this clear that index + can be negative in which case the index is calculated from the end. + If that index calculation is less than zero it is truncated to zero. + + For example (``dbg_PyList_Insert_Negative_Index()`` in ``src/cpy/Containers/DebugContainers.c``): + + .. code-block:: c + + PyObject *container = PyList_New(0); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + // Insert at end + if (PyList_Insert(container, -1L, value)) { + assert(0); + } + assert(Py_REFCNT(value) == 2); + PyObject *get_item; + // PyList_Insert at -1 actually inserts at 0. + assert(PyList_GET_SIZE(container) == 1L); + get_item = PyList_GET_ITEM(container, 0L); + assert(get_item == value); + +.. index:: + single: List; Py_BuildValue() + +.. _chapter_containers_and_refcounts.lists.Py_BuildValue: + +``Py_BuildValue()`` +------------------- + +As with tuples :ref:`chapter_containers_and_refcounts.tuples.Py_BuildValue` is a very convenient way to +create lists. +``Py_BuildValue("[O]", value);`` will increment the refcount of value and this can, potentially, leak. + +.. index:: + single: PyList_GetItem() + single: List; PyList_GetItem() + single: PyList_GET_ITEM() + single: List; PyList_GET_ITEM() + single: PyList_GetItemRef() + single: List; PyList_GetItemRef() + pair: Getters; List + pair: Getters; List + +.. _chapter_containers_and_refcounts.lists.Getters: + +List Getters +--------------------- + +There are these APIS for getting an item from a list: + +* `PyList_GetItem()`_ This is very similar to `PyTuple_GetItem()`_. It returns a borrowed reference and will error + if the supplied container is not list or the index is negative or out of range. +* `PyList_GET_ITEM()`_ This is very similar to `PyTuple_GET_ITEM()`_. It returns a borrowed reference and there is + no error checking for the index being in range. + The type checking is performed as an assertion if Python is built in + `debug mode `_ or + `with assertions `_. + If not the results are undefined. +* `PyList_GetItemRef()`_ [From Python 3.13 onwards]. + Like `PyList_GetItem()`_ but his returns a new *strong* (i.e. incremented) reference to the existing object. + +.. index:: single: List; API Summary + +Summary +---------------------- + +* `PyList_SetItem()`_ and `PyList_SET_ITEM()`_ *steal* references. +* `PyList_SetItem()`_ and `PyList_SET_ITEM()`_ behave differently when replacing an existing, different, value. +* `PyList_SetItem()`_ and `PyList_SET_ITEM()`_ behave differently when replacing the *same* value. + In particular `PyList_SetItem()`_ can produce undefined behaviour. +* If `PyList_SetItem()`_ errors it will decrement the reference count of the given value which can produce undefined + behaviour. +* `PyList_Append()`_ Increments the reference count of the given object and thus may leak. +* `PyList_Insert()`_ Increments the reference count of the given object and thus may leak. +* `Py_BuildValue()`_ Increments the reference count of the given object and thus may leak. + + +.. Links, mostly to the Python documentation: + +.. Setters + +.. _PyDict_SetItem(): https://docs.python.org/3/c-api/dict.html#c.PyDict_SetItem +.. _PyDict_SetDefault(): https://docs.python.org/3/c-api/dict.html#c.PyDict_SetDefault +.. _PyDict_SetDefaultRef(): https://docs.python.org/3/c-api/dict.html#c.PyDict_SetDefaultRef + +.. Getters + +.. _PyDict_GetItem(): https://docs.python.org/3/c-api/dict.html#c.PyDict_GetItem +.. _PyDict_GetItemRef(): https://docs.python.org/3/c-api/dict.html#c.PyDict_GetItemRef +.. _PyDict_GetItemWithError(): https://docs.python.org/3/c-api/dict.html#c.PyDict_GetItemWithError + +.. Deleters + +.. _PyDict_Pop(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Pop +.. _PyDict_DelItem(): https://docs.python.org/3/c-api/dict.html#c.PyDict_DelItem + +.. Iterators + +.. _PyDict_Items(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Items +.. _PyDict_Keys(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Keys +.. _PyDict_Values(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Values +.. _PyDict_Next(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Next + +.. Other + +.. _PyDict_Check(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Check +.. _PyObject_Hash(): https://docs.python.org/3/c-api/object.html#c.PyObject_Hash + +.. index:: + single: Dictionary + pair: Documentation Lacunae; Dictionaries + + +.. _chapter_containers_and_refcounts.dictionaries: + +----------------------- +Dictionaries +----------------------- + +This section describes how reference counts are affected when building and accessing dictionaries. + +.. index:: + single: PyDict_SetItem() + single: Dictionary; PyDict_SetItem() + +.. _chapter_containers_and_refcounts.dictionaries.setitem: + +``PyDict_SetItem()`` +-------------------- + +The Python documentation for `PyDict_SetItem()`_ is incomplete. +`PyDict_SetItem()`_ changes the key and value reference counts according to these rules: + +* If the key exists in the dictionary then key's reference count remains the same. +* If the key does *not* exist in the dictionary then its reference count will be incremented. +* The value's reference count will always be incremented. +* If the key exists in the dictionary then the previous value reference count will be decremented before the value + is replaced by the new value (and the new value reference count is incremented). + See :ref:`chapter_containers_and_refcounts.discarded`. + If the key exists in the dictionary and the value is the same then this means, effectively, that reference counts of + both key and value remain unchanged. + +.. warning:: + + If either the key or the value are NULL this will segfault. + See ``dbg_PyDict_SetItem_NULL_key()`` and ``dbg_PyDict_SetItem_NULL_value()`` in + ``src/cpy/Containers/DebugContainers.c``. + +This code illustrates `PyDict_SetItem()`_ with ``assert()`` showing the reference count: + +.. code-block:: c + + PyObject *container = PyDict_New(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + assert(Py_REFCNT(key) == 1); + assert(Py_REFCNT(value_a) == 1); + /* Insert a new key value. */ + if (PyDict_SetItem(container, key, value_a)) { + assert(0); + } + assert(Py_REFCNT(key) == 2); + assert(Py_REFCNT(value_a) == 2); + +Now replace the value with another value: + +.. code-block:: c + + /* Replace a value for the key. */ + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + assert(Py_REFCNT(key) == 2); + assert(Py_REFCNT(value_a) == 2); + assert(Py_REFCNT(value_b) == 1); + if (PyDict_SetItem(container, key, value_b)) { + assert(0); + } + assert(Py_REFCNT(key) == 2); + assert(Py_REFCNT(value_a) == 1); + assert(Py_REFCNT(value_b) == 2); + +Now replace the value with the same value, reference counts remain the same: + +.. code-block:: c + + /* Replace with the same value for the key. */ + if (PyDict_SetItem(container, key, value_b)) { + assert(0); + } + assert(Py_REFCNT(key) == 2); + assert(Py_REFCNT(value_a) == 1); + assert(Py_REFCNT(value_b) == 2); + +.. _chapter_containers_and_refcounts.dictionaries.setitem.failure: + +``PyDict_SetItem()`` Failure +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyDict_SetItem()`_ can fail for the following reasons: + +* The container is not a dictionary (or a sub-class of a dictionary, see `PyDict_Check()`_). +* The key is not hashable (`PyObject_Hash()`_ returns -1). +* If either the key or the value is NULL this will cause a SIGSEGV (or some other disaster). + These are checked with asserts if Python is built in + `debug mode `_ or + `with assertions `_. + +For code and tests see: + +* C: in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_SetItem_*()`` +* CPython: in ``src/cpy/RefCount/cRefCount.c``: + * ``test_PyDict_SetItem_*()`` +* Python: in ``tests/unit/test_c_ref_count.py``: + * ``test_PyDict_SetItem_increments()`` + +.. note:: + + In ``src/cpy/Containers/DebugContainers.c`` there are failure tests that cause a SIGSEGV if ``ACCEPT_SIGSEGV`` + is non zero. + ``ACCEPT_SIGSEGV`` is defined in ``src/cpy/Containers/DebugContainers.h``. + +.. index:: + pair: Documentation Lacunae; PyDict_SetDefault() + +.. _chapter_containers_and_refcounts.dictionaries.setdefault: + +``PyDict_SetDefault()`` +------------------------ + +`PyDict_SetDefault()`_ is equivalent to the Python method +`dict.setdefault() `_ in Python. +The C function signature is: + +.. code-block:: c + + PyObject *PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj); + +The idea is that if the key exists then the appropriate value is returned and the default value is unused. +If the key does *not* exist then the default value is inserted into the dictionary and returned. + +If the Default Value is Unused +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the key already exists in the dictionary the reference counts of the key, existing value and default value are +unchanged so the return value is a borrowed reference (see :ref:`chapter_refcount.borrowed`). + +If the Default Value is Used +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the key does *not* exist in the dictionary the reference counts of the key and default value are incremented. + +These reference count changes are not particularly clear from the official Python documentation. + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_SetDefault_default_unused()`` + * ``dbg_PyDict_SetDefault_default_used()`` +* CPython, in ``src/cpy/RefCount/cRefCount.c``. + ``test_PyDict_SetDefault_default_unused()`` + ``test_PyDict_SetDefault_default_used()`` +* Python, pytest, in ``tests.unit.test_c_ref_count``: + * ``test_PyDict_SetDefault_default_unused()`` + * ``test_PyDict_SetDefault_default_used()`` + + +.. index:: + single: PyDict_SetDefaultRef() + single: Dictionary; PyDict_SetDefaultRef() + pair: Documentation Lacunae; PyDict_SetDefaultRef() + +``PyDict_SetDefaultRef()`` [Python 3.13+] +----------------------------------------- + +`PyDict_SetDefaultRef()`_ sets a default value and retrieves the actual value. +This is new in Python 3.13+. +The C function signature is: + +.. code-block:: c + + int PyDict_SetDefaultRef( + PyObject *dictionary, PyObject *key, PyObject *default_value, PyObject **result + ); + +``*result`` +^^^^^^^^^^^ + +Any previous ``*result`` is always *abandoned* (see :ref:`chapter_containers_and_refcounts.abandoned`). +To emphasise, there is no decrementing the reference count of the existing value (if any). +This is important as the following code snippet shows: + +.. code-block:: c + + PyObject *result; /* Refers to an arbitrary memory location. */ + /* Now if PyDict_SetDefaultRef() were to attempt to Py_DECREF(result) + * the results will be undefined. + */ + PyDict_SetDefaultRef(container, key, default_value, &result); + +.. index:: + single: Dictionary; PyDict_SetDefaultRef(); Key Exists + +Key Exists +^^^^^^^^^^ + +If the key already exists in the dictionary `PyDict_SetDefaultRef()`_ returns 1. +The reference counts are changed as follows: + +- key: unchanged. +- value: incremented by one +- default_value: unchanged. + +``*result`` is equal to the stored value. + +For example: + +.. code-block:: c + + PyObject *container = PyDict_New(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *val = new_unique_string(__FUNCTION__, NULL); + /* At this point the reference counts are: + * key: 1 + * val: 1 + */ + // Set the key/value + PyDict_SetItem(container, key, val); + /* At this point the reference counts are: + * key: 2 + * val: 2 + */ + // Create a default value and a result. + PyObject *default_value = new_unique_string(__FUNCTION__, NULL); + PyObject *result = NULL; + /* At this point the reference counts are: + * default_value: 1 + * result: N/A + */ + PyDict_SetDefaultRef(container, key, default_value, &result); + /* Now the reference counts are: + * key: 2 + * val: 3 + * default_value: 1 + * result: 3 as it equals val. + */ + Py_DECREF(container); + /* Now the reference counts are: + * key: 1 + * val: 2 + * default_value: 1 + * result: 2 as it equals val. + */ + + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_SetDefaultRef_default_unused()`` +* CPython, in ``src/cpy/RefCount/cRefCount.c``. + ``test_PyDict_SetDefaultRef_default_unused()`` +* Python, pytest, in ``tests.unit.test_c_ref_count``: + * ``test_PyDict_SetDefaultRef_default_unused()`` + +.. index:: + single: Dictionary; PyDict_SetDefaultRef(); Key Does not Exist + +Key Does not Exist +^^^^^^^^^^^^^^^^^^^ + +If the key does not exists in the dictionary `PyDict_SetDefaultRef()`_ returns 0. +The reference counts are changed as follows: + +- key: incremented by one. +- default_value: incremented by *two*. The rationale is one increment as the default_value is inserted into + the dictionary then a second increment as the default_value is 'returned' as ``*result``. + +``*result`` is equal to the default_value. + +For example: + +.. code-block:: c + + PyObject *container = PyDict_New(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + // Create a default value and a result. + PyObject *default_value = new_unique_string(__FUNCTION__, NULL); + PyObject *result = NULL; + /* At this point the reference counts are: + * key: 1 + * default_value: 1 + * result: N/A + */ + PyDict_SetDefaultRef(container, key, default_value, &result); + /* Now the reference counts are: + * key: 2 + * default_value: 3 + * result: 3 as it equals default_value. + */ + Py_DECREF(container); + /* Now the reference counts are: + * key: 1 + * default_value: 2 + * result: 2 as it equals default_value. + */ + +These reference count changes are not particularly clear from the official Python documentation. + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_SetDefaultRef_default_used()`` +* CPython, in ``src/cpy/RefCount/cRefCount.c``. + ``test_PyDict_SetDefaultRef_default_used()`` +* Python, pytest, in ``tests.unit.test_c_ref_count``: + * ``test_PyDict_SetDefaultRef_default_used()`` + +.. index:: + single: Dictionary; PyDict_SetDefaultRef(); Failure + +Failure +^^^^^^^ + +.. todo:: + + PyDict_SetDefaultRef() failure modes. + + +.. index:: + single: Dictionary; PyDict_GetItem() + pair: Getters; Dictionary + +``PyDict_GetItem()`` +----------------------------------------- + +`PyDict_GetItem()`_ returns a borrowed reference (:ref:`chapter_refcount.borrowed`) to an existing value or ``NULL`` if +the key does not exist in the dictionary. + +.. warning:: + + If the key is ``NULL`` this will segfault. + See ``dbg_PyDict_GetItem_key_NULL()`` in ``src/cpy/Containers/DebugContainers.c``. + + +.. index:: + single: Dictionary; PyDict_GetItemRef() + pair: Getters; Dictionary + +``PyDict_GetItemRef()`` [Python 3.13+] +----------------------------------------- + +`PyDict_GetItemRef()`_ gets a new strong reference to a value from a dictionary. + +The C signature is: + +.. code-block:: c + + int PyDict_GetItemRef(PyObject *p, PyObject *key, PyObject **result); + +Key is Present +^^^^^^^^^^^^^^ + +If the key is in the dictionary then increment the reference count of the value and set ``*result`` to the value. +The reference count of the key is unchanged. +The function returns 1. + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_GetItemRef()`` +* CPython, in ``src/cpy/RefCount/cRefCount.c``. + ``test_PyDict_SetDefaultRef_default_used()`` +* Python, pytest, in ``tests.unit.test_c_ref_count``: + * ``test_PyDict_SetDefaultRef_default_used()`` + +.. index:: + single: Dictionary; PyDict_SetDefaultRef(); Failure + pair: Setters; Dictionary + +Failure +^^^^^^^ + +.. todo:: + + PyDict_GetItemRef() failure modes. + +.. index:: + single: Dictionary; PyDict_Pop() + +``PyDict_Pop()`` [Python 3.13+] +----------------------------------------- + +`PyDict_Pop()`_ removes a specific value. +This is new in Python 3.13+. +The C function signature is: + +.. code-block:: c + + int int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result); + +``*result`` +^^^^^^^^^^^ + +Any previous ``*result`` is always *abandoned* (see :ref:`chapter_containers_and_refcounts.abandoned`). +To emphasise, there is no decrementing the reference count of the existing value (if any). +This is important as the following code snippet shows: + +.. code-block:: c + + PyObject *result; /* Refers to an arbitrary memory location. */ + /* Now if PyDict_Pop() were to attempt to Py_DECREF(result) + * the results will be undefined. + */ + PyDict_Pop(container, key, &result); + + +.. index:: + single: Dictionary; PyDict_Pop(); Key Exists + +Key Exists +^^^^^^^^^^ + +If the key already exists in the dictionary `PyDict_Pop()`_ returns 1. +The reference counts are changed as follows: + +- key: decremented by one +- value: unchanged. + +``*result`` is equal to the stored value. + +For example: + +.. code-block:: c + + PyObject *container = PyDict_New(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *val = new_unique_string(__FUNCTION__, NULL); + /* At this point the reference counts are: + * key: 1 + * val: 1 + */ + // Set the key/value + PyDict_SetItem(container, key, val); + /* At this point the reference counts are: + * key: 2 + * val: 2 + */ + PyObject *result; + PyDict_Pop(container, key, &result); + /* Now the reference counts are: + * key: 1 + * val: 2 + * result: 2 as it equals val. + */ + +.. code-block:: c + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_Pop_key_present()`` +* CPython, in ``src/cpy/RefCount/cRefCount.c``. + ``test_PyDict_Pop_key_present()`` +* Python, pytest, in ``tests.unit.test_c_ref_count``: + * ``test_PyDict_Pop_key_present()`` + +.. index:: + single: Dictionary; PyDict_Pop(); Key Does not Exist + +Key Does not Exist +^^^^^^^^^^^^^^^^^^^ + +If the key does not exists in the dictionary `PyDict_Pop()`_ returns 0. +The reference counts are changed as follows: + +- key: unchanged. + +``*result`` is set to ``NULL``, any previous value is *abandoned* +(see :ref:`chapter_containers_and_refcounts.abandoned`). + +For example: + +.. code-block:: c + + PyObject *container = PyDict_New(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + // Create a default value and a result. + PyObject *dummy_value = new_unique_string(__FUNCTION__, NULL); + PyObject *result = dummy_value; + /* At this point the reference counts are: + * key: 1 + * dummy_value: 1 + * result: 1 as it is equal to dummy_value. + */ + PyDict_Pop(container, key, &result); + /* Now the reference counts are: + * key: 1 + * dummy_value: 1 + * result: is NULL. + */ + Py_DECREF(container); + /* No change in the reference counts. */ + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_Pop_key_absent()`` +* CPython, in ``src/cpy/RefCount/cRefCount.c``. + ``test_PyDict_Pop_key_absent()`` +* Python, pytest, in ``tests.unit.test_c_ref_count``: + * ``test_PyDict_Pop_key_absent()`` + + +.. index:: + single: Dictionary; PyDict_Pop(); Failure + +Failure +^^^^^^^ + +.. todo:: + + Finish Dictionary ``PyDict_Pop()`` Failure + + +.. index:: + single: Dictionary; Other APIs + +Other APIs +---------- + +This section describes other dictionary APIs that are simple to describe and have no complications. + +.. note:: + + There are no tests for many of these APIs in the current version of this project. + +.. todo:: + + Finish Dictionary "Other APIs" + + +.. index:: + single: Dictionary; PyDict_GetItemWithError() + +.. _chapter_containers_and_refcounts.dictionaries.pydict_getitemwitherror: + +``PyDict_GetItemWithError()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`PyDict_GetItemWithError()`_ gets a new *borrowed* reference to a value from a dictionary or NULL. +Unlike `PyDict_GetItem()`_ this will set an exception if appropriate. + +The C signature is: + +.. code-block:: c + + PyObject *PyDict_GetItemWithError(PyObject *p, PyObject *key); + +Currently, the only failure mode is if the first argument is not a dictionary. + +For code and tests see: + +* C, in ``src/cpy/Containers/DebugContainers.c``: + * ``dbg_PyDict_GetItemWithError_fails()`` + +.. index:: + single: Dictionary; PyDict_DelItem() + +``PyDict_DelItem()`` +^^^^^^^^^^^^^^^^^^^^ + +`PyDict_DelItem()`_ removes a specific value if it exists. The reference count of the value will be decremented. +The key must be hashable; if it isn’t a ``TypeError`` is set. +If key is not in the dictionary a ``KeyError`` is set. +Returns 0 on success or -1 on failure in which case an exception will have been set. + +The C function signature is: + +.. code-block:: c + + int PyDict_DelItem(PyObject *p, PyObject *key); + +.. todo:: + + Complete ``PyDict_DelItem()`` with code examples. + +.. index:: + single: Dictionary; PyDict_Items() + +``PyDict_Items()`` +^^^^^^^^^^^^^^^^^^^^ + +`PyDict_Items()`_ returns a *new* Python list containing *new* tuples of (key, value). +Each key and value will have their reference count incremented. +That is to say calling ``Py_DECREF`` on the result will change all the reference counts within the dictionary to their +previous values. + +The C function signature is: + +.. code-block:: c + + Pyobject *PyDict_Items(PyObject *p); + +.. todo:: + + Complete ``PyDict_Items()`` with code examples. + +.. index:: + single: Dictionary; PyDict_Keys() + +``PyDict_Keys()`` +^^^^^^^^^^^^^^^^^^^^ + +`PyDict_Keys()`_ returns a *new* Python list containing all the keys. +Each key will have its reference count incremented. +That is to say calling ``Py_DECREF`` on the result will change all the reference counts within the dictionary to their +previous values. + +The C function signature is: + +.. code-block:: c + + Pyobject *PyDict_Keys(PyObject *p); + +.. todo:: + + Complete ``PyDict_Keys()`` with code examples. + +.. index:: + single: Dictionary; PyDict_Values() + +``PyDict_Values()`` +^^^^^^^^^^^^^^^^^^^^ + +`PyDict_Values()`_ returns a *new* Python list containing all the values. +Each value will have its reference count incremented. +That is to say calling ``Py_DECREF`` on the result will change all the reference counts within the dictionary to their +previous values. + +The C function signature is: + +.. code-block:: c + + Pyobject *PyDict_Values(PyObject *p); + +.. todo:: + + Complete ``PyDict_Values()`` with code examples. + +.. index:: + single: Dictionary; PyDict_Next() + +``PyDict_Next()`` +^^^^^^^^^^^^^^^^^^^^ + +`PyDict_Next()`_ is the standard way of iterating through all the keys and values. +Each key and value will is a *borrowed* reference. +The C function signature is: + +.. code-block:: c + + int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); + +.. todo:: + + Complete ``PyDict_Next()`` with code examples. + +.. index:: + single: Dictionary; Py_BuildValue() + +``Py_BuildValue()`` +^^^^^^^^^^^^^^^^^^^^ + +`Py_BuildValue()`_ is a very convenient way to create dictionaries. +``Py_BuildValue("{OO}", key, value);`` will increment the refcount of the key and value and this can, +potentially, leak. + +.. Links, mostly to the Python documentation: + +.. Setters + +.. _PySet_Add(): https://docs.python.org/3/c-api/set.html#c.PySet_Add +.. _PySet_Discard(): https://docs.python.org/3/c-api/set.html#c.PySet_Discard +.. _PySet_Pop(): https://docs.python.org/3/c-api/set.html#c.PySet_Pop + +.. _chapter_containers_and_refcounts.sets: + +.. index:: + single: Set + +----------------------- +Sets +----------------------- + +The set API is simple with no real difficulties for the user. + +.. _chapter_containers_and_refcounts.sets.pyset_add: + +.. index:: + single: PySet_Add() + single: Set; PySet_Add() + pair: Setters; Set + +``PySet_Add()`` +-------------------- + +`PySet_Add()`_ is fairly straightforward. +The set will increment the reference count of the value if it is *not* already in the set. + +This returns 0 on success or -1 on failure in which case an exception will have been set which will be: + +* A ``SystemError`` if set is not an instance of set or its subtype. +* A ``TypeError`` if the key is unhashable. +* A ``MemoryError`` if there is no room to grow. + +Here is an example: + +.. code-block:: c + + PyObject *container = PySet_New(NULL); + /* Py_REFCNT(container) is 1 */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* Py_REFCNT(value) is 1 */ + int ret_val = PySet_Add(container, value); + assert(ret_val == 0); + /* Py_REFCNT(value) is now 2 */ + + /* Clean up. */ + Py_DECREF(container); + /* Py_REFCNT(value) is now 1 */ + Py_DECREF(value); + +When adding something that already exists in the set does *not* increment the reference count: + +.. code-block:: c + + PyObject *container = PySet_New(NULL); + /* Py_REFCNT(container) is 1 */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* Py_REFCNT(value) is 1 */ + int ret_val = PySet_Add(container, value); + assert(ret_val == 0); + /* Py_REFCNT(value) is now 2 */ + + /* Add duplicate. */ + ret_val = PySet_Add(container, value); + assert(ret_val == 0); + /* Py_REFCNT(value) is still 2 */ + + /* Clean up. */ + Py_DECREF(container); + /* Py_REFCNT(value) is now 1 */ + Py_DECREF(value); + +.. _chapter_containers_and_refcounts.sets.pyset_discard: + +.. index:: + single: PySet_Discard() + single: Set; PySet_Discard() + +``PySet_Discard()`` +-------------------- + +`PySet_Discard()`_ is also fairly straightforward. +The set will discard (:ref:`chapter_containers_and_refcounts.discarded`) the value thus: + +.. code-block:: c + + PyObject *container = PySet_New(NULL); + /* Py_REFCNT(container) is 1 */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* Py_REFCNT(value) is 1 */ + int ret_val = PySet_Add(container, value); + assert(ret_val == 0); + /* Py_REFCNT(value) is now 2 */ + + /* Discard. */ + ret_val = PySet_Discard(container, value); + assert(ret_val == 1); + /* Py_REFCNT(value) is now 1 */ + + /* Clean up. */ + Py_DECREF(container); + /* Py_REFCNT(value) is still 1 */ + Py_DECREF(value); + +.. _chapter_containers_and_refcounts.sets.pyset_pop: + +.. index:: + single: PySet_Pop() + single: Set; PySet_Pop() + +``PySet_Pop()`` +-------------------- + +`PySet_Pop()`_ will return a *new* reference to the existing object and it is up to caller to decrement the +reference count appropriately. + +For example, the reference counts work as follows: + +.. code-block:: c + + PyObject *container = PySet_New(NULL); + /* Py_REFCNT(container) is 1 */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* Py_REFCNT(value) is 1 */ + int ret_val = PySet_Add(container, value); + assert(ret_val == 0); + /* Py_REFCNT(value) is now 2 */ + + /* Pop. */ + PyObject *popped_value = PySet_Pop(container); + assert(popped_value == value); + /* Py_REFCNT(value) is still 2 */ + + /* Clean up. */ + Py_DECREF(container); + /* Py_REFCNT(value) is still 2 so need to double Py_DECREF calls. */ + Py_DECREF(value); + Py_DECREF(value); + +So this is how `PySet_Pop()`_ might be used in practice: + +.. code-block:: c + + + void add_to_set(PyObject *container) { + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* Py_REFCNT(value) is 1 */ + int ret_val = PySet_Add(container, value); + assert(ret_val == 0); + /* Py_REFCNT(value) is now 2 */ + /* Decrement our local value */ + Py_DECREF(value); + /* Now the container has the only reference to the value. */ + } + + void pop_from_set(PyObject *container) { + PyObject *value = PySet_Pop(container); + /* Do something with the value... */ + + /* Then as we 'own' value we need to free it. */ + Py_DECREF(value); + } + + PyObject *container = PySet_New(NULL); + add_to_set(container); + pop_from_set(container); + /* Clean up. */ + Py_DECREF(container); + +-------------- +Summary +-------------- + +This summarises where the Python documentation is wrong, absent or misleading and where you might get into trouble. + +.. index:: + single: Tuple; Summary + pair: Tuple; Gotchas + single: Tuple; PyTuple_SetItem() + single: Tuple; PyTuple_SET_ITEM() + single: Tuple; PyTuple_Pack() + single: Tuple; Py_BuildValue() + single: Tuple; PyTuple_GET_ITEM() + +Tuples +-------------- + +- `PyTuple_SetItem()`_ will lead to a SIGSEGV if the existing value is the same as the inserted value. + See :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem`. +- On failure `PyTuple_SetItem()`_ will decrement the reference count of the given value and this might well lead to a + SIGSEGV. + See :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SetItem.failures`. +- `PyTuple_SET_ITEM()`_ will lead to a memory leak when replacing an existing value as it abandons the previous + reference. + See :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM`. +- `PyTuple_SET_ITEM()`_ does no error checking so is capable of writing to arbitrary memory locations on error. + See :ref:`chapter_containers_and_refcounts.tuples.PyTuple_SET_ITEM.failures`. +- Both `PyTuple_SetItem()`_ and `PyTuple_SET_ITEM()`_ accept ``NULL`` as the value argument and behave as documented. +- `PyTuple_Pack()`_ will leak the given arguments. See :ref:`chapter_containers_and_refcounts.tuples.PyTuple_Pack`. +- `Py_BuildValue()`_ will leak the given arguments when used with the ``"O"`` argument. + See :ref:`chapter_containers_and_refcounts.tuples.Py_BuildValue`. +- `PyTuple_GET_ITEM()`_ can try to access arbitrary memory locations if the arguments are wrong. + See :ref:`chapter_containers_and_refcounts.tuples.Getters`. + + +.. index:: + single: List; Summary + pair: List; Gotchas + single: List; PyList_SetItem() + single: List; PyList_SET_ITEM() + single: List; PyList_Append() + single: List; PyList_Insert() + single: List; Py_BuildValue() + single: List; PyList_GET_ITEM() + +Lists +-------------- + +- `PyList_SetItem()`_ will lead to a SIGSEGV if the existing value is the same as the inserted value. + See :ref:`chapter_containers_and_refcounts.lists.PyList_SetItem`. +- On failure `PyList_SetItem()`_ will decrement the reference count of the given value and this might well lead to a + SIGSEGV. + See :ref:`chapter_containers_and_refcounts.lists.PyList_SET_ITEM.failures`. +- `PyList_SET_ITEM()`_ will lead to a memory leak when replacing an existing value as it abandons the previous + reference. + See :ref:`chapter_containers_and_refcounts.lists.PyList_SET_ITEM`. +- `PyList_SET_ITEM()`_ does no error checking so is capable of writing to arbitrary memory locations on error. + See :ref:`chapter_containers_and_refcounts.lists.PyList_SET_ITEM.failures`. +- Both `PyList_SetItem()`_ and `PyList_SET_ITEM()`_ accept ``NULL`` as the value argument and behave as documented. +- `PyList_Append()`_ will increment the reference count of the given argument and so may leak. + `PyList_Append()`_ uses `PyList_SET_ITEM()`_ in its implementation so those appropriate warnings apply. + See :ref:`chapter_containers_and_refcounts.lists.PyList_Append` + and :ref:`chapter_containers_and_refcounts.lists.PyList_SET_ITEM`. +- `PyList_Insert()`_ is poorly documented in the official Python documentation. + See :ref:`chapter_containers_and_refcounts.lists.PyList_Insert`. +- `Py_BuildValue()`_ will leak the given arguments when used with the ``"O"`` argument. + Simlar to tuples, describe here: :ref:`chapter_containers_and_refcounts.tuples.Py_BuildValue`. +- `PyList_GET_ITEM()`_ can try to access arbitrary memory locations if the arguments are wrong. + See :ref:`chapter_containers_and_refcounts.lists.Getters`. + + +.. index:: + single: Dictionary; Summary + pair: Dictionary; Gotchas + single: Dictionary; PyDict_SetItem() + single: Dictionary; PyDict_SetDefault() + +Dictionaries +-------------- + +- `PyDict_SetItem()`_ generally increments the reference count of the given key/value however this API + has some complicated rules about key/value reference counts that are not described in the Python documentation. + See :ref:`chapter_containers_and_refcounts.dictionaries.setitem`. +- With `PyDict_SetItem()`_ if either the key or value is ``NULL`` there will be a SIGSEGV. + See :ref:`chapter_containers_and_refcounts.dictionaries.setitem`. +- Otherwise `PyDict_SetItem()`_ has generally graceful behaviour on failure. + See :ref:`chapter_containers_and_refcounts.dictionaries.setitem.failure`. +- `PyDict_SetDefault()`_ has generally graceful behaviour on failure. + See :ref:`chapter_containers_and_refcounts.dictionaries.setdefault`. +- `PyDict_GetItemWithError()`_ is incorrectly implemented or documented. + See :ref:`chapter_containers_and_refcounts.dictionaries.pydict_getitemwitherror`. + +.. todo:: + + Complete the summary of dictionary APIs and their documentation lacunae. + +Sets +-------------- + +The set API is much cleaner than the others and contains few gotchas. +In summary: + +- `PySet_Add()`_ will increment the reference count of the value if it is not already in the set. + See :ref:`chapter_containers_and_refcounts.sets.pyset_add`. +- `PySet_Discard()`_ will decrement the reference count of the value and returns it. + It is up to the caller to decrement the reference count of the returned value. + See :ref:`chapter_containers_and_refcounts.sets.pyset_discard`. +- `PySet_Pop()`_ does not decrement the reference count of the returned value, it is + *abandoned* (see :ref:`chapter_containers_and_refcounts.abandoned`). + It is up to the caller to decrement the reference count of the returned value. + See :ref:`chapter_containers_and_refcounts.sets.pyset_pop`. + + +.. Example footnote [#]_. + +.. rubric:: Footnotes + +.. [#] The official `Python documentation `_ categorises tuples and lists + as *sequence objects* because they support the `Sequence Protocol `_ + and dictionaries and sets as *container objects* because they support the + `Mapping Protocol `_. + In this chapter I use looser language by describing all four as *containers*. + +.. [#] ``Py_BuildValue`` can not create a set, only a tuple, list or dictionary. diff --git a/doc/sphinx/source/context_manager.rst b/doc/sphinx/source/context_manager.rst new file mode 100644 index 0000000..2dc3c94 --- /dev/null +++ b/doc/sphinx/source/context_manager.rst @@ -0,0 +1,543 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + +.. _context manger: https://docs.python.org/3/glossary.html#term-context-manager> +.. _\__enter__(): https://docs.python.org/3/library/stdtypes.html#contextmanager.__enter__ +.. _\__exit__(): https://docs.python.org/3/library/stdtypes.html#contextmanager.__exit__ + +.. _chapter_context_manager: + +.. index:: + single: Context Managers + +*************************** +Context Managers +*************************** + +This chapter describes how to write +for your C objects. + +.. index:: + single: Context Managers; C Functions + +=========================== +C Functions +=========================== + +This is a summary of what is required for the C functions implementing a `context manger`_. +The is no specific ``tp_...`` slot for the context manager functions ``__enter__`` and ``__exit__``, instead they are added +to the object as named, looked up, Python methods. + +.. index:: + single: Context Managers; __enter__ + +-------------------------------------- +``__enter__`` +-------------------------------------- + +The C function must, at least, increment the reference count of ``self`` and +return ``self``: + +.. code-block:: c + + static PyObject * + ContextManager_enter(ContextManager *self, PyObject *Py_UNUSED(args)) { + /* Stuff here. */ + Py_INCREF(self); + return (PyObject *)self; + } + +.. index:: + single: Context Managers; __exit__ + +-------------------------------------- +``__exit__`` +-------------------------------------- + +The `__exit__()`_ function is declared thus. +It takes three arguments so ``METH_VARARGS`` is used. +The three arguments are each ``None`` if no exception has been raised within +the ``with`` block. +If an exception *has* been raised within the ``with`` block then the +three arguments are the exception type, value and the traceback object. + +The return value of the ``__exit__`` method tells the interpreter whether +any exception should be suppressed. +If the function returns ``False`` then the exception should be +propagated. +This is the common case. +If the function returns ``True`` then the exception should be +suppressed and execution continues with the statement immediately +after the ``with`` statement. + +The exit method is defined in C thus. +Note that there is no change to +the reference count, that is all done appropriately by the +CPython interpreter: + +.. code-block:: c + + static PyObject * + ContextManager_exit(ContextManager *self, PyObject *args) { + /* Stuff. */ + Py_RETURN_FALSE; + } + +-------------------------------------- +Method declarations +-------------------------------------- + +Note that `__enter__()`_ is declared with ``METH_NOARGS`` and `__exit__()`_ is declared with ``METH_VARARGS``: + +.. code-block:: c + + static PyMethodDef ContextManager_methods[] = { + /* ... */ + {"__enter__", (PyCFunction) ContextManager_enter, METH_NOARGS, + PyDoc_STR("__enter__() -> ContextManager")}, + {"__exit__", (PyCFunction) ContextManager_exit, METH_VARARGS, + PyDoc_STR("__exit__(exc_type, exc_value, exc_tb) -> bool")}, + /* ... */ + {NULL, NULL, 0, NULL} /* sentinel */ + }; + +================================= +Understanding the Context Manager +================================= + +What is worth understanding is the way that reference counts are incremented and +decremented and the interplay between your C code and the CPython interpreter. + +.. index:: + single: Context Managers; Without target + +---------------------------------- +A Context Manager Without a Target +---------------------------------- + +Take this simple code: + +.. code-block:: python + + from cPyExtPatt import cCtxMgr + + with cCtxMgr.ContextManager(): + pass + +.. + def test_very_simple(): + + Gives: + + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + ContextManager_exit STRT REFCNT = 1 + ContextManager_exit DONE REFCNT = 1 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4413491472 + +The sequence of reference count changes are as follows: + +#. Creating the ``cCtxMgr.ContextManager()`` calls ``ContextManager_new`` which makes the + reference count 1. +#. The ``with`` statement causes the CPython interpreter to increment the reference count + (to 2) and then call ``__enter__`` that is implemented in our C function + ``ContextManager_enter``. +#. Our ``ContextManager_enter`` function increments the reference count, so it is now 3. +#. As the context manager ends the ``with`` statement the CPython interpreter + decrements the reference count *twice* to the value 1. + The logic is: + + #. Decrement the reference count once as we are exiting the ``with`` statement. The reference count is now 2. + #. Did the ``with`` statement have a target? If not, as in this case, then decrement the reference count once more. The reference count is now 1. + +#. after the ``pass`` statement the CPython interpreter then calls ``__exit__`` which is implemented in our function + ``ContextManager_exit``. + This does not change the reference count which remains at 1. +#. As the context manager goes out of scope the CPython interpreter decrements the reference + count to 0 and then calls our C function ``ContextManager_dealloc`` with a reference count + of 0 and that frees the object. + + +.. index:: + single: Context Managers; With target + +---------------------------------- +A Context Manager With a Target +---------------------------------- + +The importance of the ``with`` statement having a target is because a context manager +can be used like this: + +.. code-block:: python + + from cPyExtPatt import cCtxMgr + + with cCtxMgr.ContextManager() as context: + pass + # context survives here with a reference count of 1. + # This will be decremented when context goes out of scope. + # For example on a function return. + +.. + def test_simple(): + + Gives: + + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4413491440 + +In this case the ``context`` survives the ``with`` statement and is available to any following code. +So step 4 above would only decrement the reference count once leaving it with a reference count of 2. +The ``context`` variable reference count will be finally decremented when it goes out of scope, for +example on a function return. + +The sequence of reference count changes are now as follows: + +#. As above, the reference count becomes 1. +#. As above, the reference count becomes 2. +#. As above, the reference count becomes 3. +#. As the context manager ends the ``with`` statement the CPython interpreter + decrements the reference count just *once* to the value 2 as there *is* a target, called ``context`` in this case. +#. After the ``pass`` statement the CPython interpreter then calls ``__exit__`` which is implemented in our function + ``ContextManager_exit``. + This does not change the reference count which remains at 2. +#. As the context manager goes out of scope the CPython interpreter decrements the reference + count to 1. + This ensures the survival of ``context`` after the ``with`` block. +#. When ``context`` goes out of scope, say on a function return or a ``del`` statement the + CPython interpreter decrements the reference count to 0 and then calls our C function + ``ContextManager_dealloc`` which frees the object. + +.. index:: + single: Context Managers; Minimal in C + +=============================== +Minimal Context Manager in C +=============================== + +Here is the minimal, complete, C code that implements context manager. +There is example code in ``src/cpy/CtxMgr/cCtxMgr.c`` and tests in +``tests/unit/test_c_ctxmgr.py`` + +First the object declaration, allocation and de-allocation functions: + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + + #include "Python.h" + + typedef struct { + PyObject_HEAD + } ContextManager; + + static ContextManager * + ContextManager_new(PyObject *Py_UNUSED(arg)) { + return PyObject_New(ContextManager, &ContextManager_Type);; + } + + static void + ContextManager_dealloc(ContextManager *self) { + PyObject_Del(self); + } + +----------------------------------------- +``__enter__`` and ``__exit__`` Methods +----------------------------------------- + +The ``__enter__`` and ``__exit__`` methods: + +.. code-block:: c + + static PyObject * + ContextManager_enter(ContextManager *self, PyObject *Py_UNUSED(args)) { + Py_INCREF(self); + return (PyObject *)self; + } + + static PyObject * + ContextManager_exit(ContextManager *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { + Py_RETURN_FALSE; + } + + static PyMethodDef ContextManager_methods[] = { + {"__enter__", (PyCFunction) ContextManager_enter, METH_VARARGS, + PyDoc_STR("__enter__() -> ContextManager")}, + {"__exit__", (PyCFunction) ContextManager_exit, METH_VARARGS, + PyDoc_STR("__exit__() -> bool")}, + {NULL, NULL, 0, NULL} /* sentinel */ + }; + + +----------------------------------------- +Type Declaration +----------------------------------------- + +The type declaration: + +.. code-block:: c + + static PyTypeObject ContextManager_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "cObject.ContextManager", + .tp_basicsize = sizeof(ContextManager), + .tp_dealloc = (destructor) ContextManager_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = ContextManager_methods, + .tp_new = (newfunc) ContextManager_new, + }; + +----------------------------------------- +Module +----------------------------------------- + +Finally the module: + +.. code-block:: c + + PyDoc_STRVAR(module_doc, "Example of a context manager."); + + static struct PyModuleDef cCtxMgr = { + PyModuleDef_HEAD_INIT, + .m_name = "cCtxMgr", + .m_doc = module_doc, + .m_size = -1, + }; + + PyMODINIT_FUNC + PyInit_cCtxMgr(void) { + PyObject *m = NULL; + m = PyModule_Create(&cCtxMgr); + if (m == NULL) { + goto fail; + } + if (PyType_Ready(&ContextManager_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "ContextManager", (PyObject *) &ContextManager_Type)) { + goto fail; + } + return m; + fail: + Py_XDECREF(m); + return NULL; + } + +----------------------------------------- +Setup +----------------------------------------- + +This code is added to the ``setup.py`` file: + +.. code-block:: python + + Extension(f"{PACKAGE_NAME}.cCtxMgr", sources=['src/cpy/CtxMgr/cCtxMgr.c', ], + include_dirs=['/usr/local/include', ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +And can be used thus: + +.. code-block:: python + + from cPyExtPatt import cCtxMgr + + with cCtxMgr.ContextManager(): + pass + +----------------------------------------- +Testing +----------------------------------------- + +The actual code in ``src/cpy/CtxMgr/cCtxMgr.c`` contains extra trace reporting that confirms the reference counts and +(no) memory leakage. + +This can be run with: + +.. code-block:: bash + + $ pytest tests/unit/test_c_ctxmgr.py -vs + +This test: + +.. code-block:: python + + def test_very_simple(): + print() + with cCtxMgr.ContextManager(): + pass + +Gives this output: + +.. code-block:: text + + tests/unit/test_c_ctxmgr.py::test_very_simple + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + ContextManager_exit STRT REFCNT = 1 + ContextManager_exit DONE REFCNT = 1 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546088432 + +This test: + +.. code-block:: python + + def test_simple(): + print() + with cCtxMgr.ContextManager() as context: + assert sys.getrefcount(context) == 3 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == cCtxMgr.BUFFER_LENGTH + assert sys.getrefcount(context) == 2 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == 0 + del context + +Gives this output: + +.. code-block:: text + + tests/unit/test_c_ctxmgr.py::test_simple + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + + +This test: + +.. code-block:: python + + def test_memory(): + proc = psutil.Process() + print() + print(f'RSS START: {proc.memory_info().rss:12,d}') + for i in range(8): + print(f'RSS START {i:5d}: {proc.memory_info().rss:12,d}') + with cCtxMgr.ContextManager() as context: + print(f'RSS START CTX: {proc.memory_info().rss:12,d}') + # Does not work in the debugger due to introspection. + # assert sys.getrefcount(context) == 3 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == cCtxMgr.BUFFER_LENGTH + print(f'RSS END CTX: {proc.memory_info().rss:12,d}') + # Does not work in the debugger due to introspection. + # assert sys.getrefcount(context) == 2 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == 0 + del context + print(f'RSS END {i:5d}: {proc.memory_info().rss:12,d}') + print(f'RSS END: {proc.memory_info().rss:12,d}') + +Gives this output: + +.. code-block:: text + + tests/unit/test_c_ctxmgr.py::test_memory + RSS START: 300,032,000 + RSS START 0: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 0: 300,048,384 + RSS START 1: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 1: 300,048,384 + RSS START 2: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 2: 300,048,384 + RSS START 3: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 3: 300,048,384 + RSS START 4: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 4: 300,048,384 + RSS START 5: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 5: 300,048,384 + RSS START 6: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 6: 300,048,384 + RSS START 7: 300,048,384 + ContextManager_new DONE REFCNT = 1 + ContextManager_enter STRT REFCNT = 2 + ContextManager_enter DONE REFCNT = 3 + RSS START CTX: 300,048,384 + RSS END CTX: 300,048,384 + ContextManager_exit STRT REFCNT = 2 + ContextManager_exit DONE REFCNT = 2 + ContextManager_dealloc STRT REFCNT = 0 + ContextManager_dealloc DONE REFCNT = 4546096048 + RSS END 7: 300,048,384 + RSS END: 300,048,384 diff --git a/doc/sphinx/source/cpp.rst b/doc/sphinx/source/cpp.rst index df66e83..fa4eafc 100644 --- a/doc/sphinx/source/cpp.rst +++ b/doc/sphinx/source/cpp.rst @@ -1,6 +1,9 @@ .. _cpp_and_: +.. index:: + single: C++ + ******************************************** Using C++ With CPython Code ******************************************** @@ -9,6 +12,8 @@ Using C++ can take a lot of the pain out of interfacing CPython code, here are s .. toctree:: - cpp_and_cpython - cpp_and_unicode - cpp_and_numpy + cpp/cpp_and_cpython + cpp/cpp_and_placement_new + cpp/cpp_and_unicode + cpp/cpp_and_numpy + cpp/cpp_and_buffer_protocol diff --git a/doc/sphinx/source/cpp/cpp_and_buffer_protocol.rst b/doc/sphinx/source/cpp/cpp_and_buffer_protocol.rst new file mode 100644 index 0000000..f200c9f --- /dev/null +++ b/doc/sphinx/source/cpp/cpp_and_buffer_protocol.rst @@ -0,0 +1,37 @@ +.. toctree:: + :maxdepth: 2 + +.. _cpp_and_buffer_protocol: + +.. index:: + single: C++; Buffer Protocol + +==================================== +C++ and the Python Buffer Protocol +==================================== + +Python's buffer protocol is a general purpose wrapper around data structures that contain +homogeneous types with a regular structure. +Examples are numpy ``ndarrays``, PIL images and Python types such as ``bytes``, ``bytearray`` and ``array.array`` types. + +The buffer protocol is described in `PEP 3118 `_. + +The great advantage of this is that it uses a shared memory model so that the data can be passed between Python or C++ +without copying. + +It is fairly straightforward to create a C++ wrapper class around the buffer protocol. + + +.. todo:: + + Complete the Buffer Protocol chapter with examples from RaPiVot and the C++ wrapper code. + +----------- +References: +----------- + +* Python documentation on objects that support the + `Buffer protocol `_. +* Python standard library for the `array module `_. + + diff --git a/doc/sphinx/source/cpp/cpp_and_cpython.rst b/doc/sphinx/source/cpp/cpp_and_cpython.rst new file mode 100644 index 0000000..3323275 --- /dev/null +++ b/doc/sphinx/source/cpp/cpp_and_cpython.rst @@ -0,0 +1,1270 @@ +.. highlight:: cpp + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + +.. _cpp_and_cpython: + +.. index:: + single: C++; PyObject* Wrappers + single: C++; References + single: C++; Reference Counts + +============================================ +C++ RAII Wrappers Around ``PyObject*`` +============================================ + +It is sometimes useful to wrap up a ``PyObject*`` in a class that will manage the reference count. +Here is a base class that shows the general idea, it takes a ``PyObject *`` and provides: + +* Construction with a ``PyObject *`` and access this with ``operator PyObject*() const``. +* ``PyObject **operator&()`` to reset the underlying pointer, for example when using it with + ``PyArg_ParseTupleAndKeywords``. +* Decrementing the reference count on destruction (potentially freeing the object). + +.. code-block:: cpp + + /** General wrapper around a PyObject*. + * This decrements the reference count on destruction. + */ + class DecRefDtor { + public: + DecRefDtor(PyObject *ref) : m_ref { ref } {} + Py_ssize_t ref_count() const { return m_ref ? Py_REFCNT(m_ref) : 0; } + // Allow setting of the (optional) argument with PyArg_ParseTupleAndKeywords + PyObject **operator&() { + Py_XDECREF(m_ref); + m_ref = NULL; + return &m_ref; + } + // Access the argument + operator PyObject*() const { return m_ref; } + // Test if constructed successfully from the new reference. + explicit operator bool() { return m_ref != NULL; } + ~DecRefDtor() { Py_XDECREF(m_ref); } + protected: + PyObject *m_ref; + }; + +.. index:: + single: C++; Borrowed PyObject* Wrappers + single: C++; Borrowed References + +------------------------------------------------- +C++ RAII Wrapper for a Borrowed ``PyObject*`` +------------------------------------------------- + +There are two useful sub-classes, one for borrowed references, one for new references which are intended to be temporary. Using borrowed references: + +.. code-block:: cpp + + /** Wrapper around a PyObject* that is a borrowed reference. + * This increments the reference count on construction and + * decrements the reference count on destruction. + */ + class BorrowedRef : public DecRefDtor { + public: + BorrowedRef(PyObject *borrowed_ref) : DecRefDtor(borrowed_ref) { + Py_XINCREF(m_ref); + } + }; + +This can be used with borrowed references as follows: + +.. code-block:: cpp + + void function(PyObject *obj) { + BorrowedRef(obj); // Increment reference here. + // ... + } // Decrement reference here. + + +.. index:: + single: C++; New PyObject* Wrappers + single: C++; New References + +------------------------------------------------- +C++ RAII Wrapper for a New ``PyObject*`` +------------------------------------------------- + +Here is a sub-class that wraps a new reference to a ``PyObject *`` and ensures it is free'd when the wrapper goes out of scope: + +.. code-block:: cpp + + /** Wrapper around a PyObject* that is a new reference. + * This owns the reference so does not increment it on construction but + * does decrement it on destruction. + */ + class NewRef : public DecRefDtor { + public: + NewRef(PyObject *new_ref) : DecRefDtor(new_ref) {} + }; + +This new reference wrapper can be used as follows: + +.. code-block:: cpp + + void function() { + NewRef(PyLongFromLong(9)); // New reference here. + // Use static_cast(NewRef) ... + } // Decrement the new reference here. + + +.. _cpp_and_cpython.handling_default_arguments: + +.. index:: + single: Parsing Arguments Example; Helper Class + single: Parsing Arguments Example; Default Mutable Arguments + single: Default Arguments; C++ + single: Default Arguments, Immutable; C++ + single: Default Arguments, Mutable; C++ + +============================================ +Handling Default Arguments +============================================ + +Handling default, possibly mutable, arguments in a pythonic way is described here: +:ref:`cpython_default_mutable_arguments`. +It is quite complicated to get it right but C++ can ease the pain with a generic class to simplify handling default +arguments in CPython functions. + +The actual code is in ``src/cpy/ParseArgs/cParseArgsHelper.cpp`` but here it is, simplified to its essentials: + +.. code-block:: cpp + + class DefaultArg { + public: + DefaultArg(PyObject *new_ref) : m_arg(NULL), m_default(new_ref) {} + /// Allow setting of the (optional) argument with + /// PyArg_ParseTupleAndKeywords + PyObject **operator&() { + m_arg = NULL; + return &m_arg; + } + /// Access the argument or the default if default. + operator PyObject *() const { + return m_arg ? m_arg : m_default; + } + PyObject *obj() const { + return m_arg ? m_arg : m_default; + } + /// Test if constructed successfully from the new reference. + explicit operator bool() { return m_default != NULL; } + protected: + PyObject *m_arg; + PyObject *m_default; + }; + +.. index:: + single: Parsing Arguments Example; Default Immutable Arguments + single: Default Arguments, Immutable; C++ + +--------------------------- +Immutable Default Arguments +--------------------------- + +Suppose we have the Python function equivalent to the Python function: + +.. code-block:: python + + def parse_defaults_with_helper_class( + encoding_m: str = "utf-8", + the_id_m: int = 1024, + log_interval_m: float = 8.0): + return encoding_m, the_id_m, log_interval_m + +Here it is in C: + +.. code-block:: cpp + + static PyObject * + parse_defaults_with_helper_class(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + PyObject *ret = NULL; + /* Initialise default arguments. */ + static DefaultArg encoding_c(PyUnicode_FromString("utf-8")); + static DefaultArg the_id_c(PyLong_FromLong(DEFAULT_ID)); + static DefaultArg log_interval_c(PyFloat_FromDouble(DEFAULT_FLOAT)); + + /* Check that the defaults are non-NULL i.e. succesful. */ + if (!encoding_c || !the_id_c || !log_interval_c) { + return NULL; + } + + static const char *kwlist[] = {"encoding", "the_id", "log_interval", NULL}; + /* &encoding etc. accesses &m_arg in DefaultArg because of PyObject **operator&() */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", + const_cast(kwlist), + &encoding_c, &the_id_c, &log_interval_c)) { + return NULL; + } + + PY_DEFAULT_CHECK(encoding_c, PyUnicode_Check, "str"); + PY_DEFAULT_CHECK(the_id_c, PyLong_Check, "int"); + PY_DEFAULT_CHECK(log_interval_c, PyFloat_Check, "float"); + + /* + * Use encoding, the_id, must_log from here on as PyObject* since we have + * operator PyObject*() const ... + * + * So if we have a function: + * set_encoding(PyObject *obj) { ... } + */ + // set_encoding(encoding); + /* ... */ + + /* Py_BuildValue("O") increments the reference count. */ + ret = Py_BuildValue("OOO", encoding_c.obj(), the_id_c.obj(), log_interval_c.obj()); + return ret; + } + +The full code is in ``src/cpy/cParseArgsHelper.cpp`` and the tests in ``tests/unit/test_c_parse_args_helper.py``. + +Here is an example test: + +.. code-block:: python + + @pytest.mark.parametrize( + 'args, expected', + ( + ( + (), + ('utf-8', 1024, 8.0), + ), + ( + ('Encoding', 4219, 16.0), + ('Encoding', 4219, 16.0), + ), + ), + ) + def test_parse_defaults_with_helper_class(args, expected): + assert cParseArgsHelper.parse_defaults_with_helper_class(*args) == expected + +.. index:: + single: Parsing Arguments Example; Default Mutable Arguments + single: Default Arguments, Mutable; C++ + +------------------------- +Mutable Default Arguments +------------------------- + +The same class can be used for mutable arguments. +The following emulates this Python function: + +.. code-block:: python + + def parse_mutable_defaults_with_helper_class(obj, default_list=[]): + default_list.append(obj) + return default_list + +Here it is in C: + +.. code-block:: c + + /** Parse the args where we are simulating mutable default of an empty list. + * This uses the helper class. + * + * This is equivalent to: + * + * def parse_mutable_defaults_with_helper_class(obj, default_list=[]): + * default_list.append(obj) + * return default_list + * + * This adds the object to the list and returns None. + * + * This imitates the Python way of handling defaults. + */ + static PyObject *parse_mutable_defaults_with_helper_class(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + static DefaultArg list_argument_c(PyList_New(0)); + + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &list_argument_c)) { + goto except; + } + PY_DEFAULT_CHECK(list_argument_c, PyList_Check, "list"); + + /* Your code here...*/ + + /* Append the first argument to the second. + * PyList_Append() increments the refcount of arg_0. */ + if (PyList_Append(list_argument_c, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; + } + + /* Success. */ + assert(!PyErr_Occurred()); + /* This increments the default or the given argument. */ + Py_INCREF(list_argument_c); + ret = list_argument_c; + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; + } + +The code is in ``src/cpy/ParseArgs/cParseArgsHelper.cpp``. + +Here are some tests from ``tests/unit/test_c_parse_args_helper.py``. +Firstly establish the known Python behaviour: + +.. code-block:: python + + def test_parse_mutable_defaults_with_helper_class_python(): + """A local Python equivalent of cParseArgsHelper.parse_mutable_defaults_with_helper_class().""" + + def parse_mutable_defaults_with_helper_class(obj, default_list=[]): + default_list.append(obj) + return default_list + + result = parse_mutable_defaults_with_helper_class(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = parse_mutable_defaults_with_helper_class(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = parse_mutable_defaults_with_helper_class(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_class(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_class(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = parse_mutable_defaults_with_helper_class(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 + +And now the equivalent in C: + +.. code-block:: python + + from cPyExtPatt import cParseArgsHelper + + def test_parse_mutable_defaults_with_helper_class_c(): + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_class(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_class(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 + + +.. index:: + single: C++; Homogeneous Containers + single: C++; Project PyCppContainers + +============================================ +Homogeneous Python Containers and C++ +============================================ + +Here are some useful generic functions that can convert homogeneous Python containers to and from their C++ STL +equivalents in this project: +`Python/C++ homogeneous containers on GitHub `_ +The project uses a mixture of templates and code generation to provide 300+ functions to convert to and from +C++ and Python containers. + +Here is the introduction to that project: + +Python is well known for it's ability to handle *heterogeneous* data in containers such as lists like: + +.. code-block:: python + + >>> l = [1, 2.0, "some string", ] + +But what if you need to interact with C++ containers such as ``std::vector`` that require *homogeneous* data types? + +This project is about converting Python containers such as ``list``, ``tuple``, ``dict``, ``set``, ``frozenset`` +containing homogeneous types such as ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str`` or user defined +types to and from their C++ equivalent. + +Here is a general example of the use of this library where Python data needs to be passed to and from a C++ library and +those results need to be presented in Python. +Like this, visually: + +.. code-block:: text + + Python | This Library (C++/Python) | Some C++ Library + ------------------- . ----------------------------- . ------------------ + | . . + Get Python data . . + | . . + \---------------------->\ . + . | . + . Convert Python data to C++ . + . | . + . \---------------------------->\ + . . | + . . Process C++ data + . . | + . /<----------------------------/ + . | . + . Convert C++ data to Python . + . | . + /<----------------------/ . + | . . + Process Python data . . + | . . + +Here is a, problematic, example of how to do this: + +.. raw:: latex + + \pagebreak + +-------------------------------- +A Problematic Example +-------------------------------- + +Suppose that you have a Python list of floats and need to pass it to a C++ library that expects a +``std::vector``. +If the result of that call modifies the C++ vector, or creates a new one, you need to return a Python list of floats +from the result. + +Your C++ code might look like this (error checking omitted): + +.. code-block:: cpp + + PyObject *example(PyObject *op) { + std::vector vec; + // Populate the vector, function to be defined... + write_to_vector(op, vec); + // Do something in C++ with the vector + // ... + // Convert the vector back to a Python list. + // Function to be defined... + return read_from_vector(vec); + } + +What should the implementation of ``write_to_vector()`` and ``read_from_vector()`` look like? + +The answer seems fairly simple; firstly ``write_to_vector`` converting a Python list to a C++ ``std::vector`` +with Pythons C-API: + +.. code-block:: cpp + + void write_to_vector(PyObject *op, std::vector &vec) { + vec.clear(); + for (Py_ssize_t i = 0; i < PyList_Size(op); ++i) { + vec.push_back(PyFloat_AsDouble(PyList_GET_ITEM(op, i))); + } + } + +And the inverse, ``read_from_vector`` creating a new Python list from a C++ ``std::vector``: + +.. code-block:: cpp + + PyObject *read_from_vector(const std::vector &vec) { + PyObject *ret = PyList_New(vec.size()); + for (size_t i = 0; i < vec.size(); ++i) { + PyList_SET_ITEM(ret, i, PyFloat_FromDouble(vec[i])); + } + return ret; + } + + +There is no error handling shown here, and all errors would be runtime errors. + +However if you need to support other object types, say lists of ``int``, ``str``, ``bytes`` then each one needs a pair +of hand written functions; Python to C++ and C++ to Python. +It gets worse when you want to support other containers such as ``tuple``, ``list``, ``set``, ``frozenset``, ``dict``. +You end up with hundreds of functions, all individually named, to handle all the combinations. +Then you have to write individual conversion functions, and their tests, for all the combinations of object types *and* +containers. + +This is tedious and error prone and hard to extend in the general case. + +Why This Project +========================= + +This project simplifies the problem of converting data from Python to C++ and vice versa *in general*. + +The project makes extensive use of C++ templates, partial template specialisation and code generation to dramatically +reduce the amount of hand maintained code. +It also converts many runtime errors to compile time errors. + +The types and containers this library supports are: + +.. list-table:: **Supported Object types.** + :widths: 20 10 45 + :header-rows: 1 + + * - **C++ Type** + - **Python Type** + - **Notes** + * - ``bool`` + - ``True``, ``False`` + - + * - ``long`` + - ``int`` + - + * - ``double`` + - ``float`` + - + * - ``std::complex`` + - ``complex`` + - + * - ``std::vector`` + - ``bytes`` + - ``bytearray`` is not supported as we need hashable types for ``set`` and ``dict`` containers. + * - ``std::string`` + - ``str`` + - Specifically a ``PyUnicode_1BYTE_KIND`` [#f1]_. + `Python documentation `_ + * - ``std::u16string`` + - ``str`` + - Specifically a ``PyUnicode_2BYTE_KIND``. + `Python documentation `_ + * - ``std::u32string`` + - ``str`` + - Specifically a ``PyUnicode_4BYTE_KIND``. + `Python documentation `_ + +Used in these containers: + +.. list-table:: **Supported Containers.** + :widths: 50 50 + :header-rows: 1 + + * - **C++ Container** + - **Python Equivalent** + * - ``std::vector`` + - Either a ``tuple`` or ``list`` + * - ``std::list`` + - Either a ``tuple`` or ``list`` + * - ``std::unordered_set`` + - Either a ``set`` or ``frozenset`` + * - ``std::unordered_map`` + - ``dict`` + * - ``std::map`` + - ``dict`` + +The number of possible conversion functions is worse than the cartesian product of the types and containers as in the +case of a dict the types can appear as either a key or a value. + +Supporting all these conversions would normally require 352 conversion functions to be written, tested and documented +[#f2]_ . + +This project simplifies this by using a mix of C++ templates and code generators to reduce this number to just +**six** hand written templates for all 352 cases. + +Using This Library +======================== + +Python to C++ +------------------- + +Using the library is as simple as this, suppose you have data in Python that needs to be passed to a C++ library: + +.. code-block:: text + + Python | This Library (C++/Python) | Some C++ Library + ------------------- . ----------------------------- . ------------------ + | . . + Python data source . . + | . . + \---------------------->\ . + . | . + . Convert Python data to C++ . + . | . + . \------------------------------>\ + . . | + . . Process C++ data + +The C++ code using this library looks like this: + +C++ Code +^^^^^^^^^^^^^^^^^^^ + +.. code-block:: cpp + + #include "python_convert.h" + + // Create a Python list of floats: [21.0, 42.0, 3.0] + PyObject *op = Py_BuildValue("[ddd]", 21.0, 42.0, 3.0); + + // Create the C++ vector that we want to convert this data to... + std::vector cpp_vector; + + // The template specialisation will automatically invoke the appropriate + // function call. + // It will be a compile time error if the container/type function + // is not supported. + // At run time this will return zero on success, non-zero on failure, + // for example if op is not a Python tuple or members of op can not be + // converted to C++ doubles. + int err = Python_Cpp_Containers::py_list_to_cpp_std_list_like(op, cpp_vector); + // Handle error checking if err is non-zero... + +.. note:: + + If you were to change the C++ container to a ``std::list`` the function call + ``py_list_to_cpp_std_list_like()`` would be the same. + Of course ``py_list_to_cpp_std_list_like()`` would then dispatch to code handling a ``std::list``. + +Another example, suppose the Python data source is a ``typing.Dict[int, str]`` and this needs to be converted to a +C++ ``std::map>`` then a function using the conversion code using this library is as simple as this: + +.. code-block:: cpp + + #include "python_convert.h" + + void convert_py_data_to_cpp(PyObject *arg) { + std::map map; + if (Python_Cpp_Containers::py_dict_to_cpp_std_map_like(arg, map)) { + // Handle error... + } else { + // Use the map... + } + } + +.. note:: + + If you were to change the C++ container to a ``std::unordered_map`` the function call + ``py_dict_to_cpp_std_map_like()`` would be the same. + Of course ``py_dict_to_cpp_std_map_like()`` would then dispatch to code handling a + ``std::unordered_map``. + + +C++ to Python +------------------- + +Suppose that you have data from a C++ library and this data needs to be represented in Python: + +.. code-block:: text + + Python | This Library (C++/Python) | Some C++ Library + ------------------- . ----------------------------- . ------------------ + . . C++ data source + . . | + . /<---------------------------/ + . | . + . Convert C++ data to Python . + . | . + /<----------------------/ . + | . . + Python data . . + | . . + +The C++ code using this library looks like this: + +.. code-block:: cpp + + #include "python_convert.h" + + std::vector cpp_vector; + // Populate the C++ vector... + cpp_vector.push_back(21.0); + cpp_vector.push_back(42.0); + cpp_vector.push_back(3.0); + + // Now convert to Python. + // This will be a compile time error if the C++ type is not supported. + PyObject *op = Python_Cpp_Containers::cpp_std_list_like_to_py_list(cpp_vector); + // op is a Python list of floats: [21.0, 42.0, 3.0] + // op will be null on failure and a Python exception will have been set. + +.. note:: + + If you were to change the C++ container to a ``std::list`` the function call + ``cpp_std_list_like_to_py_list()`` would be the same. + Of course ``cpp_std_list_like_to_py_list()`` would then dispatch to code handling a ``std::list``. + +Another example, suppose the C++ data source is a ``std::map>`` and we need this a Python dict +``typing.Dict[int, str]`` then the conversion code in this library is as simple as this: + +.. code-block:: cpp + + #include "python_convert.h" + + PyObject *convert_cpp_data_to_py() { + std::map map; + // Populate map from the C++ data source + // ... + // Now convert to a Python dict: + return Python_Cpp_Containers::cpp_std_map_like_to_py_dict(map); + } + +The Hand Written Functions +============================= + +At the heart off this library here are just six non-trivial hand written functions along with a much larger of +generated functions that successively specialise these handwritten functions. +They are defined as templates in ``src/cpy/python_object_convert.h``. + +* Two C++ templates for Python ``tuple`` / ``list`` to and from ``std::list`` or ``std::vector`` for all types. +* Two C++ templates for Python ``set`` / ``frozenset`` to and from ``std::unordered_set`` for all types. +* Two C++ templates for Python ``dict`` to and from ``std::map`` or ``std::unordered_map`` for all type pairs. + +These six handwritten templates are short, simple and comprehensible. +Then, for simplicity, a Python script is used to create the final, instantiated, 352 functions. + +As an example, here how the function is developed that converts a Python list of ``float`` to and from a C++ +``std::vector`` or ``std::list``. + +First C++ to Python. + +Converting a C++ ``std::vector`` or ``std::list`` to a Python ``tuple`` or ``list`` +-------------------------------------------------------------------------------------------------------------------- + +The generic function signature looks like this: + +.. code-block:: cpp + + template< + template class ListLike, + typename T, + PyObject *(*ConvertCppToPy)(const T &), + PyObject *(*PyUnaryContainer_New)(size_t), + int(*PyUnaryContainer_Set)(PyObject *, size_t, PyObject *) + > + PyObject * + very_generic_cpp_std_list_like_to_py_unary(const ListLike &list_like) { + // Handwritten code, see "C++ to Python Implementation" below. + // ... + } + +.. list-table:: ``very_generic_cpp_std_list_like_to_py_unary()`` template parameters. + :widths: 30 70 + :header-rows: 1 + + * - Template Parameter + - Notes + * - ``ListLike`` + - The C++ container type, either a ``std::vector`` or ``std::list``. + * - ``T`` + - The C++ type of the objects in the target C++ container. + * - ``ConvertCppToPy`` + - A pointer to a function that converts any C++ ``T`` to a ``PyObject *``, for example from ``double`` -> ``float``. + The function signature is ``PyObject *ConvertCppToPy(const T&)``. + This returns NULL on failure. + * - ``PyUnaryContainer_New`` + - A pointer to a function that creates a new Python container, for example a ``list``, of a particular length. + The function signature is ``PyObject *PyUnaryContainer_New(Py_ssize_t)``. + This returns NULL on failure. + * - ``PyUnaryContainer_Set`` + - A pointer to a function that sets a ``PyObject *`` in the Python container at a given index. + The function signature is ``int PyUnaryContainer_Set(PyObject *container, size_t pos, PyObject *value))``. + This returns 0 on success. + +And the function has the following parameters. + +.. list-table:: ``very_generic_cpp_std_list_like_to_py_unary()`` parameters. + :widths: 20 20 50 + :header-rows: 1 + + * - Type + - Name + - Notes + * - ``ListLike &`` + - ``list_like`` + - The C++ list like container to read from to. + +The return value is non-NULL on success or NULL if there is a runtime error. +These errors could be: + +* ``PyObject *`` container can not be created. +* A member of the Python container can not be created from the C++ type ``T``. +* The ``PyObject *`` can not be inserted into the Python container. + +C++ to Python Implementation +-------------------------------- + +The implementation is fairly straightforward in ``src/cpy/python_object_convert.h`` (lightly edited): + +.. code-block:: cpp + + template< + template class ListLike, + typename T, + PyObject *(*ConvertCppToPy)(const T &), + PyObject *(*PyUnaryContainer_New)(size_t), + int(*PyUnaryContainer_Set)(PyObject *, size_t, PyObject *) + > + PyObject * + very_generic_cpp_std_list_like_to_py_unary(const ListLike &list_like) { + assert(!PyErr_Occurred()); + PyObject *ret = PyUnaryContainer_New(list_like.size()); + if (ret) { + size_t i = 0; + for (const auto &val: list_like) { + PyObject *op = (*ConvertCppToPy)(val); + if (!op) { + // Failure, do not need to decref the contents as that will + // be done when decref'ing the container. + // e.g. tupledealloc(): + // https://github.com/python/cpython/blob/main/Objects/tupleobject.c + PyErr_Format(PyExc_ValueError, "C++ value of can not be converted."); + goto except; + } + // PyUnaryContainer_Set wraps a function returning non-zero on error. + if (PyUnaryContainer_Set(ret, i++, op)) { // Stolen reference. + PyErr_Format(PyExc_RuntimeError, "Can not set unary value."); + goto except; + } + } + } else { + PyErr_Format( + PyExc_ValueError, + "Can not create Python container of size %ld", + list_like.size() + ); + goto except; + } + assert(!PyErr_Occurred()); + assert(ret); + goto finally; + except: + Py_XDECREF(ret); + assert(PyErr_Occurred()); + ret = NULL; + finally: + return ret; + } + +Partial Specialisation to Convert a C++ ``std::vector`` or ``std::list`` to a Python ``list``` +------------------------------------------------------------------------------------------------------- + +As an example this is specialised for a C++ ``std::vector`` and a Python ``list`` with a handwritten oneliner: + +.. code-block:: cpp + + template< + typename T, + PyObject *(*ConvertCppToPy)(const T &) + > + PyObject * + generic_cpp_std_list_like_to_py_list(const std::vector &container) { + return very_generic_cpp_std_list_like_to_py_unary< + std::vector, T, ConvertCppToPy, &py_list_new, &py_list_set + >(container); + } + +.. note:: + + The use of the function pointers to ``py_list_new``, and ``py_list_set`` that are defined in this + project namespace. + These are thin wrappers around existing functions or macros in ``"Python.h"``. + There is no error checking in these functions. + + For example: + + .. code-block:: c + + PyObject *py_list_new(size_t len) { + return PyList_New(len); + } + int py_list_set(PyObject *list_p, size_t pos, PyObject *op) { + // No error checking, always "succeeds". + PyList_SET_ITEM(list_p, pos, op); + return 0; + } + +There is a similar partial specialisation for a Python ``tuple``: + +.. code-block:: cpp + + template< + typename T, + PyObject *(*ConvertCppToPy)(const T &) + > + PyObject * + generic_cpp_std_list_like_to_py_list(const std::vector &container) { + return very_generic_cpp_std_list_like_to_py_unary< + std::vector, T, ConvertCppToPy, &py_tuple_new, &py_tuple_set + >(container); + } + +And the tuple functions are trivial and look like the list ones in the note above. +There is no error checking in these functions: + +.. code-block:: c + + PyObject *py_tuple_new(size_t len) { + return PyTuple_New(len); + } + int py_tuple_set(PyObject *tuple_p, size_t pos, PyObject *op) { + // No error checking, always "succeeds". + PyTuple_SET_ITEM(tuple_p, pos, op); + return 0; + } + +Converting a Python ``tuple`` or ``list`` to a C++ ``std::vector`` or ``std::list`` +-------------------------------------------------------------------------------------------------- + +The reverse is converting Python to C++. +This generic function that converts unary Python indexed containers (``tuple`` and ``list``) to a C++ ``std::vector`` +or ``std::list`` for any type has this signature: + +.. code-block:: cpp + + template< + template class ListLike, + typename T, + int (*PyObject_Check)(PyObject *), + T (*PyObject_Convert)(PyObject *), + int(*PyUnaryContainer_Check)(PyObject *), + Py_ssize_t(*PyUnaryContainer_Size)(PyObject *), + PyObject *(*PyUnaryContainer_Get)(PyObject *, size_t)> + int very_generic_py_unary_to_cpp_std_list_like( + PyObject *op, ListLike &list_like + ) { + // Handwritten code, see "Python to C++ Implementation" below. + // ... + } + +This template has these parameters: + +.. list-table:: ``very_generic_py_unary_to_cpp_std_list_like()`` template parameters. + :widths: 20 50 + :header-rows: 1 + + * - Template Parameter + - Notes + * - ``ListLike`` + - The C++ container type, either a ``std::vector`` or ``std::list``. + * - ``T`` + - The C++ type of the objects in the target C++ container. + * - ``PyObject_Check`` + - A pointer to a function that checks that any ``PyObject *`` in the Python container is the correct type, + for example that it is a ``bytes`` object. + The function signature is ``int PyObject_Check(PyObject *)``. + This returns non-zero if the Python object is as expected. + * - ``PyObject_Convert`` + - A pointer to a function that converts any ``PyObject *`` in the Python container to the C++ type, for example + from ``bytes`` -> ``std::vector``. + The function signature is ``T PyObject_Convert(PyObject *)``. + * - ``PyUnaryContainer_Check`` + - A pointer to a function that checks that the ``PyObject *`` argument is the correct container type, for example + a ``tuple``. + The function signature is ``int PyUnaryContainer_Check(PyObject *)``. + This returns non-zero if the Python container is not as expected. + * - ``PyUnaryContainer_Size`` + - A pointer to a function that returns the size of the Python container. + The function signature is ``Py_ssize_t PyUnaryContainer_Size(PyObject *op)``. + This returns the size of the the Python container. + * - ``PyUnaryContainer_Get`` + - A pointer to a function that gets a ``PyObject *`` from the Python container at a given index. + The function signature is ``PyObject *PyUnaryContainer_Get(PyObject *, size_t)``. + +And the function has the following parameters. + +.. list-table:: ``generic_py_unary_to_cpp_std_list_like()`` parameters. + :widths: 20 20 50 + :header-rows: 1 + + * - Type + - Name + - Notes + * - ``PyObject *`` + - ``op`` + - The Python container to read from. + * - ``ListLike &`` + - ``list_like`` + - The C++ list like container to write to. + +The return value is zero on success or non-zero if there is a runtime error. +These errors could be: + +* ``PyObject *op`` is not a container of the required type. +* A member of the Python container can not be converted to the C++ type ``T`` (``PyObject_Check`` fails). + +Python to C++ Implementation +---------------------------------- + +The implementation is fairly straightforward in ``src/cpy/python_object_convert.h`` (lightly edited): + +.. code-block:: cpp + + template< + template class ListLike, + typename T, + int (*PyObject_Check)(PyObject *), + T (*PyObject_Convert)(PyObject *), + int(*PyUnaryContainer_Check)(PyObject *), + Py_ssize_t(*PyUnaryContainer_Size)(PyObject *), + PyObject *(*PyUnaryContainer_Get)(PyObject *, size_t) + > + int very_generic_py_unary_to_cpp_std_list_like(PyObject *op, ListLike &list_like) { + assert(!PyErr_Occurred()); + int ret = 0; + list_like.clear(); + Py_INCREF(op); // Increment borrowed reference + if (!PyUnaryContainer_Check(op)) { + PyErr_Format( + PyExc_ValueError, + "Can not convert Python container of type %s", + op->ob_type->tp_name + ); + ret = -1; + goto except; + } + for (Py_ssize_t i = 0; i < PyUnaryContainer_Size(op); ++i) { + PyObject *value = PyUnaryContainer_Get(op, i); + if (!value) { + ret = -2; + goto except; + } + if (!(*PyObject_Check)(value)) { + list_like.clear(); + PyErr_Format( + PyExc_ValueError, + "Python value of type %s can not be converted", + value->ob_type->tp_name + ); + ret = -3; + goto except; + } + list_like.push_back((*PyObject_Convert)(value)); + // Check !PyErr_Occurred() which could never happen as we check first. + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + list_like.clear(); + finally: + Py_DECREF(op); // Decrement borrowed reference + return ret; + } + +Partial Specialisation to Convert a Python ``list`` to a C++ ``std::vector`` or ``std::list`` +------------------------------------------------------------------------------------------------------- + +This template can be partially specialised for converting Python *lists* of any type to C++ ``std::vector`` or ``std::list``. +This is hand written code but it is trivial by wrapping a single function call. + +In the particular case of a ``std::vector`` we can use ``.reserve()`` as an optimisations to avoid excessive re-allocations. + +.. code-block:: cpp + + template< + typename T, + int (*PyObject_Check)(PyObject *), + T (*PyObject_Convert)(PyObject *) + > + int generic_py_list_to_cpp_std_list_like( + PyObject *op, std::vector &container + ) { + // Reserve the vector, but only if it is a list. + // If it is any other Python object then ignore it as py_list_len() + // may give undefined behaviour. + // Leave it to very_generic_py_unary_to_cpp_std_list_like() to error + if (py_list_check(op)) { + container.reserve(py_list_len(op)); + } + return very_generic_py_unary_to_cpp_std_list_like< + std::vector, T, PyObject_Check, PyObject_Convert, + &py_list_check, &py_list_len, &py_list_get + >(op, container); + } + +.. note:: + + The use of the function pointers to ``py_list_check``, ``py_list_len`` and ``py_list_get`` that are defined in this + project namespace. + These are thin wrappers around existing functions or macros in ``"Python.h"``. + There is no error checking in these functions. + + For example: + + .. code-block:: c + + int py_list_check(PyObject *op) { + return PyList_Check(op); + } + Py_ssize_t py_list_len(PyObject *op) { + return PyList_Size(op); + } + PyObject *py_list_get(PyObject *list_p, size_t pos) { + return PyList_GET_ITEM(list_p, pos); + } + +There is a similar partial specialisation for the Python ``tuple``: + +.. code-block:: cpp + + template + int generic_py_tuple_to_cpp_std_list_like(PyObject *op, std::vector &container) { + // Reserve the vector, but only if it is a tuple. + // If it is any other Python object then ignore it as py_tuple_len() + // may give undefined behaviour. + // Leave it to very_generic_py_unary_to_cpp_std_list_like() to error + if (py_tuple_check(op)) { + container.reserve(py_tuple_len(op)); + } + return very_generic_py_unary_to_cpp_std_list_like< + std::vector, T, PyObject_Check, PyObject_Convert, + &py_tuple_check, &py_tuple_len, &py_tuple_get + >(op, container); + } + +The functions ``py_tuple_len`` and ``py_tuple_get`` are thin wrappers round existing functions or macros in +``"Python.h"`` as above. + +Generated Functions +============================= + +The particular function specialisations are created by a Python script that takes the cartesian product of object types +and container types and creates functions for each container/object. + +C++ to Python +---------------------------- + +For example, to convert a C++ ``std::vector`` to a Python ``list`` of ``float`` the following are created: + +A base declaration in *auto_py_convert_internal.h*: + +.. code-block:: cpp + + template + PyObject * + cpp_std_list_like_to_py_list(const std::vector &container); + +And a concrete declaration for each C++ target type ``T`` in *auto_py_convert_internal.h*: + +.. code-block:: cpp + + template <> + PyObject * + cpp_std_list_like_to_py_list(const std::vector &container); + +And the concrete definition is in *auto_py_convert_internal.cpp*, this simply calls the generic function: + +.. code-block:: cpp + + template <> + PyObject * + cpp_std_list_like_to_py_list(const std::vector &container) { + return generic_cpp_std_list_like_to_py_list< + double, &cpp_double_to_py_float + >(container); + } + +Here is the function hierarchy for converting lists to C++ ``std::vector`` or ``std::list``: +This is the function hierarchy for the code that converts C++ ``std::vector`` or ``std::list`` to Python +``list`` and ``tuple`` for all supported object types. + +.. code-block:: none + + very_generic_cpp_std_list_like_to_py_unary <-- Hand written + | + /--------------------------\ + | | Hand written partial + generic_cpp_std_list_like_to_py_list tuples... <-- specialisation for + | | std::vector + | | and std::list + | | (generally trivial). + | | + cpp_std_list_like_to_py_list ... <-- Generated + | | + /-------------------------------\ /-------\ + | | | | Generated declaration + cpp_std_list_like_to_py_list ... ... ... <-- and implementation + (one liners) + +Python to C++ +---------------------------- + +For example, to convert a Python ``list`` of ``float`` to a C++ ``std::vector`` the following are generated: + +A base declaration in *auto_py_convert_internal.h*: + +.. code-block:: cpp + + template + int + py_list_to_cpp_std_list_like(PyObject *op, std::list &container); + +And a concrete declaration for each C++ target type ``T`` in *auto_py_convert_internal.h*: + +.. code-block:: cpp + + template <> + int + py_list_to_cpp_std_list_like(PyObject *op, std::list &container); + + +And the concrete definition is in *auto_py_convert_internal.cpp*: + +.. code-block:: cpp + + template <> + int + py_list_to_cpp_std_list_like(PyObject *op, std::vector &container) { + return generic_py_list_to_cpp_std_list_like< + double, &py_float_check, &py_float_to_cpp_double + >(op, container); + } + + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +This is the function hierarchy for the code that converts Python ``list`` and ``tuple`` to C++ ``std::vector`` or +``std::list`` for all supported object types. + +.. code-block:: none + + very_generic_py_unary_to_cpp_std_list_like <-- Hand written + | + /--------------------------\ + | | Hand written partial + generic_py_list_to_cpp_std_list_like tuples... <-- specialisation for + | | std::vector + | | and std::list + | | (generally trivial). + | | + py_list_to_cpp_std_list_like ... <-- Generated + | | + /-------------------------------\ /-------\ + | | | | Generated declaration + py_list_to_cpp_std_list_like ... ... ... <-- and implementation + (one liners) + + +More information can be found from this project +`Python/C++ homogeneous containers on GitHub `_. + + +.. rubric:: Footnotes + +.. [#f1] We are currently targeting C++14 so we use ``std::string`` which is defined as ``std::basic_string``. + C++20 allows a stricter, and more desirable, definition ``std::basic_string`` that we could use here. + See `C++ reference for std::string `_ +.. [#f2] There are six unary container pairings (``tuple`` <-> ``std::list``, ``tuple`` <-> ``std::vector``, + ``list`` <-> ``std::list``, ``list`` <-> ``std::vector``, + ``set`` <-> ``std::unordered_set``, ``frozenset`` <-> ``std::unordered_set``) with eight types + (``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str[1]``, ``str[2]``, ``str[4]``). + Each container/type combination requires two functions to give two way conversion from Python to C++ and back. + Thus 6 (container pairings) * 8 (types) * 2 (way conversion) = 96 required functions. + For ``dict`` there are two container pairings (``dict`` <-> ``std::map``, ``dict`` <-> ``std::unordered_map``) + with the eight types either of which can be the key or the value so 64 (8**2) possible variations. + Thus 2 (container pairings) * 64 (type pairs) * 2 (way conversion) = 256 required functions. + Thus is a total of 96 + 256 = 352 functions. diff --git a/doc/sphinx/source/cpp/cpp_and_numpy.rst b/doc/sphinx/source/cpp/cpp_and_numpy.rst new file mode 100644 index 0000000..e563b7a --- /dev/null +++ b/doc/sphinx/source/cpp/cpp_and_numpy.rst @@ -0,0 +1,313 @@ +.. toctree:: + :maxdepth: 2 + +.. _cpp_and_numpy: + +.. index:: + pair: C++; Numpy + +==================================== +C++ and the Numpy C API +==================================== + +`Numpy `_ is a powerful arrary based data structure with fast vector and array operations. +It has a fully featured `C API `_. +This section describes some aspects of using Numpy with C++. + +.. index:: + single: C++; Initialising Numpy + +------------------------------------ +Initialising Numpy +------------------------------------ + +The Numpy C API must be setup so that a number of static data structures are initialised correctly. +The way to do this is to call ``import_array()`` which makes a number of Python import statements so the Python +interpreter must be initialised first. This is described in detail in the +`Numpy documentation `_ +so this document just presents a cookbook approach. + +------------------------------------ +Verifying Numpy is Initialised +------------------------------------ + +``import_array()`` always returns ``NUMPY_IMPORT_ARRAY_RETVAL`` regardless of success instead we have to check the +Python error status: + +.. code-block:: cpp + + #include + #include "numpy/arrayobject.h" // Include any other Numpy headers, UFuncs for example. + + // Initialise Numpy + import_array(); + if (PyErr_Occurred()) { + std::cerr << "Failed to import numpy Python module(s)." << std::endl; + return NULL; // Or some suitable return value to indicate failure. + } + +In other running code where Numpy is expected to be initialised then ``PyArray_API`` should be non-NULL and this can be +asserted: + +.. code-block:: cpp + + assert(PyArray_API); + +.. index:: + single: C++; Numpy Initialisation Techniques + single: Numpy; C++ Initialisation Techniques + +------------------------------------ +Numpy Initialisation Techniques +------------------------------------ + +Initialising Numpy in a CPython Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Taking the simple example of a module from the +`Python documentation `_ +we can add Numpy access just by including the correct Numpy header file and calling ``import_numpy()`` in the module +initialisation code: + +.. code-block:: cpp + + #include + + #include "numpy/arrayobject.h" // Include any other Numpy headers, UFuncs for example. + + static PyMethodDef SpamMethods[] = { + ... + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static struct PyModuleDef spammodule = { + PyModuleDef_HEAD_INIT, + "spam", /* name of module */ + spam_doc, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods + }; + + PyMODINIT_FUNC + PyInit_spam(void) { + ... + assert(! PyErr_Occurred()); + import_numpy(); // Initialise Numpy + if (PyErr_Occurred()) { + return NULL; + } + ... + return PyModule_Create(&spammodule); + } + +That is fine for a singular translation unit but you have multiple translation units then each has to initialise the +Numpy API which is a bit extravagant. The following sections describe how to manage this with multiple translation +units. + +Initialising Numpy in Pure C++ Code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is mainly for development and testing of C++ code that uses Numpy. Your code layout might look something like this +where ``main.cpp`` has a ``main()`` entry point and ``class.h`` has your class declarations and ``class.cpp`` has their +implementations, like this:: + + . + └── src + └── cpp + ├── class.cpp + ├── class.h + └── main.cpp + +The way of managing Numpy initialisation and access is as follows. In ``class.h`` choose a unique name such as +``awesome_project`` then include: + +.. code-block:: cpp + + #define PY_ARRAY_UNIQUE_SYMBOL awesome_project_ARRAY_API + #include "numpy/arrayobject.h" + +In the implementation file ``class.cpp`` we do not want to import Numpy as that is going to be handled by ``main()`` +in ``main.cpp`` so we put this at the top: + +.. code-block:: cpp + + #define NO_IMPORT_ARRAY + #include "class.h" + +Finally in ``main.cpp`` we initialise Numpy: + +.. code-block:: cpp + + #include "Python.h" + #include "class.h" + + int main(int argc, const char * argv[]) { + // ... + // Initialise the Python interpreter + wchar_t *program = Py_DecodeLocale(argv[0], NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); + exit(1); + } + Py_SetProgramName(program); /* optional but recommended */ + Py_Initialize(); + // Initialise Numpy + import_array(); + if (PyErr_Occurred()) { + std::cerr << "Failed to import numpy Python module(s)." << std::endl; + return -1; + } + assert(PyArray_API); + // ... + } + +If you have multiple .h, .cpp files then it might be worth having a single .h file, say ``numpy_init.h`` with just this +in: + +.. code-block:: cpp + + #define PY_ARRAY_UNIQUE_SYMBOL awesome_project_ARRAY_API + #include "numpy/arrayobject.h" + +Then each implementation .cpp file has: + +.. code-block:: cpp + + #define NO_IMPORT_ARRAY + #include "numpy_init.h" + #include "class.h" // Class declarations + +And ``main.cpp`` has: + +.. code-block:: cpp + + #include "numpy_init.h" + #include "class_1.h" + #include "class_2.h" + #include "class_3.h" + + int main(int argc, const char * argv[]) { + // ... + import_array(); + if (PyErr_Occurred()) { + std::cerr << "Failed to import numpy Python module(s)." << std::endl; + return -1; + } + assert(PyArray_API); + // ... + } + +Initialising Numpy in a CPython Module using C++ Code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Supposing you have laid out your source code in the following fashion:: + + . + └── src + ├── cpp + │ ├── class.cpp + │ └── class.h + └── cpython + └── module.c + +This is a hybrid of the above and typical for CPython C++ extensions where ``module.c`` contains the CPython code that +allows Python to access the pure C++ code. + +The code in ``class.h`` and ``class.cpp`` is unchanged and the code in ``module.c`` is essentially the same as that of +a CPython module as described above where ``import_array()`` is called from within the ``PyInit_`` function. + + +How These Macros Work Together +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The two macros ``PY_ARRAY_UNIQUE_SYMBOL`` and ``NO_IMPORT_ARRAY`` work together as follows: + ++-----------------------------------+-------------------------------+-------------------------------+ +| | | | ++-----------------------------------+-------------------------------+-------------------------------+ +| | ``PY_ARRAY_UNIQUE_SYMBOL`` | ``PY_ARRAY_UNIQUE_SYMBOL`` | +| | NOT defined | defined as | ++-----------------------------------+-------------------------------+-------------------------------+ +| | C API is declared as: | C API is declared as: | +| ``NO_IMPORT_ARRAY`` not defined | ``static void **PyArray_API`` | ``void **`` | +| | Which makes it only available | so can be seen by other | +| | to that translation unit. | translation units. | ++-----------------------------------+-------------------------------+-------------------------------+ +| | C API is declared as: | C API is declared as: | +| ``NO_IMPORT_ARRAY`` defined | ``extern void **PyArray_API`` | ``extern void **`` | +| | so is available from another | so is available from another | +| | translation unit. | translation unit. | ++-----------------------------------+-------------------------------+-------------------------------+ + + +Adding a Search Path to a Virtual Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you are linking to the system Python this may not have numpy installed, here is a way to cope with that. +Create a virtual environment from the system python and install numpy: + +.. code-block:: bash + + python -m venv + source /bin/activate + pip install numpy + +Then in your C++ entry point add this function that manipulates ``sys.path``: + +.. code-block:: cpp + + /** Takes a path and adds it to sys.paths by calling PyRun_SimpleString. + * This does rather laborious C string concatenation so that it will work in + * a primitive C environment. + * + * Returns 0 on success, non-zero on failure. + */ + int add_path_to_sys_module(const char *path) { + int ret = 0; + const char *prefix = "import sys\nsys.path.append(\""; + const char *suffix = "\")\n"; + char *command = (char*)malloc(strlen(prefix) + + strlen(path) + + strlen(suffix) + + 1); + if (! command) { + return -1; + } + strcpy(command, prefix); + strcat(command, path); + strcat(command, suffix); + ret = PyRun_SimpleString(command); + #ifdef DEBUG + printf("Calling PyRun_SimpleString() with:\n"); + printf("%s", command); + printf("PyRun_SimpleString() returned: %d\n", ret); + fflush(stdout); + #endif + free(command); + return ret; + } + +``main()`` now calls this with the path to the virtual environment ``site-packages``: + +.. code-block:: cpp + + int main(int argc, const char * argv[]) { + wchar_t *program = Py_DecodeLocale(argv[0], NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); + exit(1); + } + // Initialise the interpreter. + Py_SetProgramName(program); /* optional but recommended */ + Py_Initialize(); + const char *multiarray_path = ""; + add_path_to_sys_module(multiarray_path); + import_array(); + if (PyErr_Occurred()) { + std::cerr << "Failed to import numpy Python module(s)." << std::endl; + return -1; + } + assert(PyArray_API); + // Your code here... + } diff --git a/doc/sphinx/source/cpp/cpp_and_placement_new.rst b/doc/sphinx/source/cpp/cpp_and_placement_new.rst new file mode 100644 index 0000000..a3f74b4 --- /dev/null +++ b/doc/sphinx/source/cpp/cpp_and_placement_new.rst @@ -0,0 +1,120 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 2 + +.. _cpp_and_placement_new: + +========================================== +A CPython Extension containing C++ Objects +========================================== + +Here is an example of using C++ classes within a CPython extension type (Python object). +It shows the various stages of construction and destruction. + +Only the important code is shown here. The complete code is in ``src/cpy/cpp/placement_new.cpp`` and the tests are +in ``tests/unit/test_c_cpp.py`` + +----------------------------------------------- +Allocation of C++ Objects and Placement new +----------------------------------------------- + +In ``src/cpy/cpp/placement_new.cpp`` there is a C++ class ``Verbose`` which: + +- Reports on ``stdout`` construction and destruction events. +- Allocates an in-memory buffer of 256MB so that the memory usage, and any leaks, will show up in the process RSS. + +We are going to create a Python extension that has a Python class that contains the C++ ``Verbose`` objects in two +ways: + +- Directly. +- With a dynamically allocated pointer. + +This will illustrate the different techniques needed for both. +Here is the CPython structure: + +.. code-block:: cpp + + typedef struct { + PyObject_HEAD + Verbose Attr; + Verbose *pAttr; + } CppCtorDtorInPyObject; + +Here is the function to allocate a new CPython object. +The important point here is that the line below: + +.. code-block:: cpp + + self = (CppCtorDtorInPyObject *) type->tp_alloc(type, 0); + +Allocates sufficient, uninitialised, space for the ``CppCtorDtorInPyObject`` object. +This will mean that both the ``Verbose Attr;`` and ``Verbose *pAttr;`` are uninitialised. +To initialise them two different techniques must be used: + +- For ``Verbose Attr;`` this must be initialised with *placement new* ``new(&self->Attr) Verbose;``. +- For ``Verbose *pAttr;`` this must be initialised with a dynamic new: ``self->pAttr = new Verbose("pAttr");``. + +Here is the complete code: + +.. code-block:: cpp + + static PyObject * + CppCtorDtorInPyObject_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { + printf("-- %s()\n", __FUNCTION__); + CppCtorDtorInPyObject *self; + self = (CppCtorDtorInPyObject *) type->tp_alloc(type, 0); + if (self != NULL) { + // Placement new used for direct allocation. + new(&self->Attr) Verbose; + self->Attr.print("Initial self->Attr"); + // Dynamically allocated new. + self->pAttr = new Verbose("pAttr"); + if (self->pAttr == NULL) { + Py_DECREF(self); + return NULL; + } else { + self->pAttr->print("Initial self->pAttr"); + } + } + return (PyObject *) self; + } + +The complimentary de-allocation function uses different deletion techniques for the two objects. + +.. code-block:: cpp + + static void + CppCtorDtorInPyObject_dealloc(CppCtorDtorInPyObject *self) { + printf("-- %s()\n", __FUNCTION__); + self->Attr.print("self->Attr before delete"); + // For self->Attr call the destructor directly. + self->Attr.~Verbose(); + self->pAttr->print("self->pAttr before delete"); + // For self->pAttr use delete. + delete self->pAttr; + Py_TYPE(self)->tp_free((PyObject *) self); + } + +The C++ ``Verbose`` class writes to ``stdout`` the stages of construction and deletion, typically the output is: + +.. code-block:: bash + + RSS start: 35,586,048 + -- CppCtorDtorInPyObject_new() + Constructor at 0x102afa840 with argument "Default" buffer len: 268435456 + Default constructor at 0x102afa840 with argument "Default" + Initial self->Attr: Verbose object at 0x102afa840 m_str: "Default" + Constructor at 0x600003158000 with argument "pAttr" buffer len: 268435456 + Initial self->pAttr: Verbose object at 0x600003158000 m_str: "pAttr" + -- CppCtorDtorInPyObject_buffer_size() + Buffer size: 536,871,116 + RSS new: 572,506,112 +536,920,064 + -- CppCtorDtorInPyObject_dealloc() + self->Attr before delete: Verbose object at 0x102afa840 m_str: "Default" + Destructor at 0x102afa840 m_str: "Default" + self->pAttr before delete: Verbose object at 0x600003158000 m_str: "pAttr" + Destructor at 0x600003158000 m_str: "pAttr" + RSS del: 35,602,432 +16,384 + RSS end: 35,602,432 +16,384 diff --git a/doc/sphinx/source/cpp_and_unicode.rst b/doc/sphinx/source/cpp/cpp_and_unicode.rst similarity index 67% rename from doc/sphinx/source/cpp_and_unicode.rst rename to doc/sphinx/source/cpp/cpp_and_unicode.rst index f683e1b..74af7b7 100644 --- a/doc/sphinx/source/cpp_and_unicode.rst +++ b/doc/sphinx/source/cpp/cpp_and_unicode.rst @@ -6,13 +6,17 @@ .. _cpp_and_unicode: +.. index:: + pair: C++; Unicode + ==================================== Python Unicode Strings and C++ ==================================== -Yes Unicode is a pain but it here to stay, particularly with Python 3. This section looks at how you can bridge between Python and C++ unicode in Python extensions. This section is only about Python 3+ and C++11 or more. +This section looks at how you can bridge between Python and C++ unicode in Python extensions. -Whilst Python is Unicode aware C++ is not, well C++11 added ``std::basic_string`` specialisations for 2 and 4 byte 'Unicode' characters but these are just containers, they have no real awareness of what they contain. +Whilst Python is Unicode aware C++ is not, well C++11 added ``std::basic_string`` specialisations for 2 and 4 byte +'Unicode' characters but these are just containers, they have no real awareness of what they contain. ------------------------------------ Basic Handling of Unicode @@ -27,7 +31,8 @@ The task here is to: This is just show that we can round-trip between the internal representations of the two languages. -Here is the despatch function that takes a single Unicode argument (note the ``"U"`` specification) and calls the appropriate handling function: +Here is the despatch function that takes a single Unicode argument (note the ``"U"`` specification) and calls the +appropriate handling function: .. code-block:: cpp @@ -36,27 +41,31 @@ Here is the despatch function that takes a single Unicode argument (note the ``" PyObject *unicode_2_to_string_and_back(PyObject *py_str); PyObject *unicode_4_to_string_and_back(PyObject *py_str); - PyObject* - unicode_to_string_and_back(PyObject * /* module */, PyObject *args) { + static PyObject * + unicode_to_string_and_back(PyObject *Py_UNUSED(module), PyObject *args) { PyObject *py_str = NULL; PyObject *ret_val = NULL; - if (PyArg_ParseTuple(args, "U", &py_str)) { - switch (PyUnicode_KIND(py_str)) { - case PyUnicode_1BYTE_KIND: - ret_val = unicode_1_to_string_and_back(py_str); - break; - case PyUnicode_2BYTE_KIND: - ret_val = unicode_2_to_string_and_back(py_str); - break; - case PyUnicode_4BYTE_KIND: - ret_val = unicode_4_to_string_and_back(py_str); - break; - default: - PyErr_Format(PyExc_ValueError, - "In %s argument is not recognised as a Unicode 1, 2, 4 byte string", - __FUNCTION__); - break; - } + if (! PyArg_ParseTuple(args, "U", &py_str)) { + return NULL; + } + unicode_dump_as_1byte_string(py_str); + std::cout << "Native:" << std::endl; + switch (PyUnicode_KIND(py_str)) { + case PyUnicode_1BYTE_KIND: + ret_val = unicode_1_to_string_and_back(py_str); + break; + case PyUnicode_2BYTE_KIND: + ret_val = unicode_2_to_string_and_back(py_str); + break; + case PyUnicode_4BYTE_KIND: + ret_val = unicode_4_to_string_and_back(py_str); + break; + default: + PyErr_Format(PyExc_ValueError, + "In %s argument is not recognised as a Unicode 1, 2, 4 byte string", + __FUNCTION__); + ret_val = NULL; + break; } return ret_val; } @@ -65,39 +74,39 @@ The three handler functions are here, they use ``std::string``, ``std::u16string .. code-block:: c - PyObject* + static PyObject * unicode_1_to_string_and_back(PyObject *py_str) { assert(PyUnicode_KIND(py_str) == PyUnicode_1BYTE_KIND); - std::string result = std::string((char*)PyUnicode_1BYTE_DATA(py_str)); + std::string result = std::string((char *) PyUnicode_1BYTE_DATA(py_str)); dump_string(result); return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, result.c_str(), result.size()); } - PyObject* + static PyObject * unicode_2_to_string_and_back(PyObject *py_str) { assert(PyUnicode_KIND(py_str) == PyUnicode_2BYTE_KIND); - // std::u16string is a std::basic_string - std::u16string result = std::u16string((char16_t*)PyUnicode_2BYTE_DATA(py_str)); + // NOTE: std::u16string is a std::basic_string + std::u16string result = std::u16string((char16_t *) PyUnicode_2BYTE_DATA(py_str)); dump_string(result); return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, result.c_str(), result.size()); } - PyObject* + static PyObject * unicode_4_to_string_and_back(PyObject *py_str) { assert(PyUnicode_KIND(py_str) == PyUnicode_4BYTE_KIND); - // std::u32string is a std::basic_string - std::u32string result = std::u32string((char32_t*)PyUnicode_4BYTE_DATA(py_str)); + // NOTE: std::u32string is a std::basic_string + std::u32string result = std::u32string((char32_t *) PyUnicode_4BYTE_DATA(py_str)); dump_string(result); return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, result.c_str(), result.size()); } -Each of these calls ``dump_string`` which is a template function: +Each of these calls ``dump_string`` which is a template function that spits out the individual character values: .. code-block:: cpp @@ -145,11 +154,13 @@ For completeness here is the module code that creates a ``cUnicode`` module with return m; } + +The full code is in ``src/cpy/cpp/cUnicode.cpp`` and the tests are in ``tests/unit/test_c_cpp.py``. Here is an example of using this module: .. code-block:: py - >>> import cUnicode + >>> from cPyExtPatt.cpp import cUnicode >>> cUnicode.show('Hello') String size: 5 word size: 1 0x00000048 72 "H" @@ -179,58 +190,55 @@ Here is an example of using this module: >>> r == s True +.. index:: + single: C++; bytes + single: C++; bytearray + ----------------------------------------------------------------------- Working with ``bytes``, ``bytearray`` and UTF-8 Unicode Arguments ----------------------------------------------------------------------- -It is fairly common to want to convert an argumennt that is ``bytes``, ``bytearray`` or UTF-8 to a ``std::string``. This function willl do just that: +It is fairly common to want to convert an argumennt that is ``bytes``, ``bytearray`` or UTF-8 to a ``std::string``. +This function will do just that: .. code-block:: c - /* Convert a PyObject to a std::string and return 0 if succesful. + /** Converting Python bytes and Unicode to and from std::string + * Convert a PyObject to a std::string and return 0 if successful. * If py_str is Unicode than treat it as UTF-8. * This works with Python 2.7 and Python 3.4 onwards. */ - int py_string_to_std_string(const PyObject *py_str, - std::string &result, - bool utf8_only=true) { + static int + py_object_to_std_string(const PyObject *py_object, std::string &result, bool utf8_only = true) { result.clear(); - if (PyBytes_Check(py_str)) { - result = std::string(PyBytes_AS_STRING(py_str)); + if (PyBytes_Check(py_object)) { + result = std::string(PyBytes_AS_STRING(py_object)); return 0; } - if (PyByteArray_Check(py_str)) { - result = std::string(PyByteArray_AS_STRING(py_str)); + if (PyByteArray_Check(py_object)) { + result = std::string(PyByteArray_AS_STRING(py_object)); return 0; } // Must be unicode then. - if (! PyUnicode_Check(py_str)) { + if (!PyUnicode_Check(py_object)) { PyErr_Format(PyExc_ValueError, "In %s \"py_str\" failed PyUnicode_Check()", __FUNCTION__); return -1; } - if (PyUnicode_READY(py_str)) { + if (PyUnicode_READY(py_object)) { PyErr_Format(PyExc_ValueError, "In %s \"py_str\" failed PyUnicode_READY()", __FUNCTION__); return -2; } - if (utf8_only && PyUnicode_KIND(py_str) != PyUnicode_1BYTE_KIND) { + if (utf8_only && PyUnicode_KIND(py_object) != PyUnicode_1BYTE_KIND) { PyErr_Format(PyExc_ValueError, "In %s \"py_str\" not utf-8", __FUNCTION__); return -3; } - // Python 3 and its minor versions (they vary) - // const Py_UCS1 *pChrs = PyUnicode_1BYTE_DATA(pyStr); - // result = std::string(reinterpret_cast(pChrs)); - #if PY_MAJOR_VERSION >= 3 - result = std::string((char*)PyUnicode_1BYTE_DATA(py_str)); - #else - // Nasty cast away constness because PyString_AsString takes non-const in Py2 - result = std::string((char*)PyString_AsString(const_cast(py_str))); - #endif + result = std::string((char *) PyUnicode_1BYTE_DATA(py_object)); return 0; } @@ -238,21 +246,19 @@ And these three do the reverse: .. code-block:: c - PyObject* + static PyObject * std_string_to_py_bytes(const std::string &str) { return PyBytes_FromStringAndSize(str.c_str(), str.size()); } - PyObject* + static PyObject * std_string_to_py_bytearray(const std::string &str) { return PyByteArray_FromStringAndSize(str.c_str(), str.size()); } - PyObject* + static PyObject * std_string_to_py_utf8(const std::string &str) { // Equivelent to: // PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, str.c_str(), str.size()); return PyUnicode_FromStringAndSize(str.c_str(), str.size()); } - - diff --git a/doc/sphinx/source/cpp_and_cpython.rst b/doc/sphinx/source/cpp_and_cpython.rst deleted file mode 100644 index a692129..0000000 --- a/doc/sphinx/source/cpp_and_cpython.rst +++ /dev/null @@ -1,495 +0,0 @@ -.. highlight:: cpp - :linenothreshold: 10 - -.. toctree:: - :maxdepth: 3 - -.. _cpp_and_cpython: - -============================================ -C++ RAII Wrappers Around ``PyObject*`` -============================================ - -It is sometimes useful to wrap up a ``PyObject*`` in a class that will manage the reference count. Here is a base class that shows the general idea, it takes a ``PyObject *`` and provides: - -* Construction with a ``PyObject *`` and access this with ``operator PyObject*() const``. -* ``PyObject **operator&()`` to reset the underlying pointer, for example when using it with ``PyArg_ParseTupleAndKeywords``. -* Decrementing the reference count on destruction (potientially freeing the object). - -.. code-block:: cpp - - /** General wrapper around a PyObject*. - * This decrements the reference count on destruction. - */ - class DecRefDtor { - public: - DecRefDtor(PyObject *ref) : m_ref { ref } {} - Py_ssize_t ref_count() const { return m_ref ? Py_REFCNT(m_ref) : 0; } - // Allow setting of the (optional) argument with PyArg_ParseTupleAndKeywords - PyObject **operator&() { - Py_XDECREF(m_ref); - m_ref = NULL; - return &m_ref; - } - // Access the argument - operator PyObject*() const { return m_ref; } - // Test if constructed successfully from the new reference. - explicit operator bool() { return m_ref != NULL; } - ~DecRefDtor() { Py_XDECREF(m_ref); } - protected: - PyObject *m_ref; - }; - -------------------------------------------------- -C++ RAII Wrapper for a Borrowed ``PyObject*`` -------------------------------------------------- - -There are two useful sub-classes, one for borrowed references, one for new references which are intended to be temporary. Using borrowed references: - -.. code-block:: cpp - - /** Wrapper around a PyObject* that is a borrowed reference. - * This increments the reference count on construction and - * decrements the reference count on destruction. - */ - class BorrowedRef : public DecRefDtor { - public: - BorrowedRef(PyObject *borrowed_ref) : DecRefDtor(borrowed_ref) { - Py_XINCREF(m_ref); - } - }; - -This can be used with borrowed references as follows: - -.. code-block:: cpp - - void function(PyObject *obj) { - BorrowedRef(obj); // Increment reference here. - // ... - } // Decrement reference here. - - -------------------------------------------------- -C++ RAII Wrapper for a New ``PyObject*`` -------------------------------------------------- - -Here is a sub-class that wraps a new reference to a ``PyObject *`` and ensures it is free'd when the wrapper goes out of scope: - -.. code-block:: cpp - - /** Wrapper around a PyObject* that is a new reference. - * This owns the reference so does not increment it on construction but - * does decrement it on destruction. - */ - class NewRef : public DecRefDtor { - public: - NewRef(PyObject *new_ref) : DecRefDtor(new_ref) {} - }; - -This new reference wrapper can be used as follows: - -.. code-block:: cpp - - void function() { - NewRef(PyLongFromLong(9)); // New reference here. - // Use static_cast(NewRef) ... - } // Decrement the new reference here. - - -============================================ -Handling Default Arguments -============================================ - -Handling default, possibly mutable, arguments in a pythonic way is described here: :ref:`cpython_default_arguments`. It is quite complicated to get it right but C++ can ease the pain with a generic class to simplify handling default arguments in CPython functions: - -.. code-block:: cpp - - class DefaultArg { - public: - DefaultArg(PyObject *new_ref) : m_arg { NULL }, m_default { new_ref } {} - // Allow setting of the (optional) argument with PyArg_ParseTupleAndKeywords - PyObject **operator&() { m_arg = NULL; return &m_arg; } - // Access the argument or the default if default. - operator PyObject*() const { return m_arg ? m_arg : m_default; } - // Test if constructed successfully from the new reference. - explicit operator bool() { return m_default != NULL; } - protected: - PyObject *m_arg; - PyObject *m_default; - }; - -Suppose we have the Python function signature of ``def function(encoding='utf8', cache={}):`` then in C/C++ we can do this: - -.. code-block:: cpp - - PyObject * - function(PyObject * /* module */, PyObject *args, PyObject *kwargs) { - /* ... */ - static DefaultArg encoding(PyUnicode_FromString("utf8")); - static DefaultArg cache(PyDict_New()); - /* Check constructed OK. */ - if (! encoding || ! cache) { - return NULL; - } - static const char *kwlist[] = { "encoding", "cache", NULL }; - if (! PyArg_ParseTupleAndKeywords(args, kwargs, "|OO", const_cast(kwlist), &encoding, &cache)) { - return NULL; - } - /* Then just use encoding, cache as if they were a PyObject* (possibly - * might need to be cast to some specific PyObject*). */ - - /* ... */ - } - -============================================ -Homogeneous Python Containers and C++ -============================================ - -Here are some useful generic functions that can convert homogeneous Python containers to and from their C++ STL equivalents. They use templates to identify the C++ type and function pointers to convert from Python to C++ objects and back. These functions must have a these characteristics on error: - -* Converting from C++ to Python, on error set a Python error (e.g with ``PyErr_SetString`` or ``PyErr_Format``) and return NULL. -* Converting from Python to C++ set a Python error and return a default C++ object (for example an empty ``std::string``). - -For illustration here are a couple of such functions that convert ``PyBytesObject*`` to and from ``std::string``: - -.. code-block:: cpp - - std::string py_bytes_to_std_string(PyObject *py_str) { - std::string r; - if (PyBytes_Check(py_str)) { - r = std::string(PyBytes_AS_STRING(py_str)); - } else { - PyErr_Format(PyExc_TypeError, - "Argument %s must be bytes not \"%s\"", - __FUNCTION__, Py_TYPE(py_str)->tp_name); - } - return r; - } - - PyObject *std_string_to_py_bytes(const std::string &str) { - return PyBytes_FromStringAndSize(str.c_str(), str.size()); - } - -We can use this for a variety of containers, first Python lists of ``bytes``. - --------------------------------------------- -Python Lists and C++ ``std::vector`` --------------------------------------------- - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python ``list`` to C++ ``std::vector`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This converts a python list to a ``std::vector``. ``ConvertToT`` is a function pointer to a function that takes a ``PyObject*`` and returns an instance of a ``T`` type. On failure to to convert a ``PyObject*`` this function should set a Python error (making ``PyErr_Occurred()`` non-NULL) and return a default ``T``. On failure this function sets ``PyErr_Occurred()`` and the return value will be an empty vector. - -.. code-block:: cpp - - template - std::vector - py_list_to_std_vector(PyObject *py_list, T (*ConvertToT)(PyObject *)) { - assert(cpython_asserts(py_list)); - std::vector cpp_vector; - - if (PyList_Check(py_list)) { - cpp_vector.reserve(PyList_GET_SIZE(py_list)); - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(py_list); ++i) { - cpp_vector.emplace(cpp_vector.end(), - (*ConvertToT)(PyList_GetItem(py_list, i))); - if (PyErr_Occurred()) { - cpp_vector.clear(); - break; - } - } - } else { - PyErr_Format(PyExc_TypeError, - "Argument \"py_list\" to %s must be list not \"%s\"", - __FUNCTION__, Py_TYPE(py_list)->tp_name); - } - return cpp_vector; - } - -If we have a function ``std::string py_bytes_to_std_string(PyObject *py_str);`` (above) we can use this thus, we have to specify the C++ template specialisation: - -.. code-block:: cpp - - std::vector result = py_list_to_std_vector(py_list, &py_bytes_to_std_string); - if (PyErr_Occurred()) { - // Handle error condition. - } else { - // All good. - } - - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -C++ ``std::vector`` to Python ``list`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -And the inverse that takes a C++ ``std::vector`` and makes a Python ``list``. ``ConvertToPy`` is a pointer to a function that takes an instance of a ``T`` type and returns a ``PyObject*``, this should return NULL on failure and set ``PyErr_Occurred()``. On failure this function sets ``PyErr_Occurred()`` and returns NULL. - -.. code-block:: cpp - - template - PyObject* - std_vector_to_py_list(const std::vector &cpp_vec, - PyObject *(*ConvertToPy)(const T&) - ) { - assert(cpython_asserts()); - PyObject *r = PyList_New(cpp_vec.size()); - if (! r) { - goto except; - } - for (Py_ssize_t i = 0; i < cpp_vec.size(); ++i) { - PyObject *item = (*ConvertToPy)(cpp_vec[i]); - if (! item || PyErr_Occurred() || PyList_SetItem(r, i, item)) { - goto except; - } - } - assert(! PyErr_Occurred()); - assert(r); - goto finally; - except: - assert(PyErr_Occurred()); - // Clean up list - if (r) { - // No PyList_Clear(). - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(r); ++i) { - Py_XDECREF(PyList_GET_ITEM(r, i)); - } - Py_DECREF(r); - r = NULL; - } - finally: - return r; - } - -If we have a function ``PyObject *std_string_to_py_bytes(const std::string &str);`` (above) we can use this thus: - -.. code-block:: cpp - - std::vector cpp_vector; - // Initialise cpp_vector... - PyObject *py_list = std_vector_to_py_list(cpp_vector, &std_string_to_py_bytes); - if (! py_list) { - // Handle error condition. - } else { - // All good. - } - ------------------------------------------------------------- -Python Sets, Frozensets and C++ ``std::unordered_set`` ------------------------------------------------------------- - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python ``set`` to C++ ``std::unordered_set`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Convert a Python ``set`` or ``frozenset`` to a ``std::unordered_set``. ``ConvertToT`` is a function pointer to a function that takes a ``PyObject*`` and returns an instance of a ``T`` type. This function should make ``PyErr_Occurred()`` true on failure to convert a ``PyObject*`` and return a default ``T``. On failure this sets ``PyErr_Occurred()`` and the return value will be an empty container. - -.. code-block:: cpp - - template - std::unordered_set - py_set_to_std_unordered_set(PyObject *py_set, T (*ConvertToT)(PyObject *)) { - assert(cpython_asserts(py_set)); - std::unordered_set cpp_set; - - if (PySet_Check(py_set) || PyFrozenSet_Check(py_set)) { - // The C API does not allow direct access to an item in a set so we - // make a copy and pop from that. - PyObject *set_copy = PySet_New(py_set); - if (set_copy) { - while (PySet_GET_SIZE(set_copy)) { - PyObject *item = PySet_Pop(set_copy); - if (! item || PyErr_Occurred()) { - PySet_Clear(set_copy); - cpp_set.clear(); - break; - } - cpp_set.emplace((*ConvertToT)(item)); - Py_DECREF(item); - } - Py_DECREF(set_copy); - } else { - assert(PyErr_Occurred()); - } - } else { - PyErr_Format(PyExc_TypeError, - "Argument \"py_set\" to %s must be set or frozenset not \"%s\"", - __FUNCTION__, Py_TYPE(py_set)->tp_name); - } - return cpp_set; - } - - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -C++ ``std::unordered_set`` to Python ``set`` or ``frozenset`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Convert a ``std::unordered_set`` to a new Python ``set`` or ``frozenset``. ``ConvertToPy`` is a pointer to a function that takes an instance of a ``T`` type and returns a ``PyObject*``, this function should return NULL on failure. On failure this function sets ``PyErr_Occurred()`` and returns NULL. - -.. code-block:: cpp - - template - PyObject* - std_unordered_set_to_py_set(const std::unordered_set &cpp_set, - PyObject *(*ConvertToPy)(const T&), - bool is_frozen=false) { - assert(cpython_asserts()); - PyObject *r = NULL; - if (is_frozen) { - r = PyFrozenSet_New(NULL); - } else { - r = PySet_New(NULL); - } - if (! r) { - goto except; - } - for (auto &iter: cpp_set) { - PyObject *item = (*ConvertToPy)(iter); - if (! item || PyErr_Occurred() || PySet_Add(r, item)) { - goto except; - } - } - assert(! PyErr_Occurred()); - assert(r); - goto finally; - except: - assert(PyErr_Occurred()); - // Clean up set - if (r) { - PySet_Clear(r); - Py_DECREF(r); - r = NULL; - } - finally: - return r; - } - ------------------------------------------------------ -Python Dicts and C++ ``std::unordered_map`` ------------------------------------------------------ - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python ``dict`` to C++ ``std::unordered_map`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Convert a Python ``dict`` to a ``std::unordered_map``. ``PyKeyConvertToK`` and ``PyKeyConvertToK`` are function pointers to functions that takes a ``PyObject*`` and returns an instance of a ``K`` or ``V`` type. On failure to convert a ``PyObject*`` this function should make ``PyErr_Occurred()`` true and return a default value. - -On failure this function will make ``PyErr_Occurred()`` non-NULL and return an empty map. - -.. code-block:: cpp - - template - std::unordered_map - py_dict_to_std_unordered_map(PyObject *dict, - K (*PyKeyConvertToK)(PyObject *), - V (*PyValConvertToV)(PyObject *) - ) { - Py_ssize_t pos = 0; - PyObject *key = NULL; - PyObject *val = NULL; - std::unordered_map cpp_map; - - if (! PyDict_Check(dict)) { - PyErr_Format(PyExc_TypeError, - "Argument \"dict\" to %s must be dict not \"%s\"", - __FUNCTION__, Py_TYPE(dict)->tp_name); - return cpp_map; - } - while (PyDict_Next(dict, &pos, &key, &val)) { - K cpp_key = (*PyKeyConvertToK)(key); - if (PyErr_Occurred()) { - cpp_map.clear(); - break; - } - V cpp_val = (*PyValConvertToV)(val); - if (PyErr_Occurred()) { - cpp_map.clear(); - break; - } - cpp_map.emplace(cpp_key, cpp_val); - } - return cpp_map; - } - -The following expects a Python dict of ``{bytes : bytes}`` and will convert it to a ``std::unordered_map``: - -.. code-block:: cpp - - std::unordered_map result; - result = py_dict_to_std_unordered_map(py_dict, - &py_bytes_to_std_string, - &py_bytes_to_std_string); - if (PyErr_Occurred()) { - // Handle failure... - } else { - // Success... - } - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -C++ ``std::unordered_map`` to Python ``dict`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This generic function converts a ``std::unordered_map`` to a new Python ``dict``. ``KeyConvertToPy``, ``ValConvertToPy`` are pointers to functions that takes an instance of a ``K`` or ``V`` type and returns a ``PyObject*``. These should return a new reference on success, NULL on failure. - -.. code-block:: cpp - - template - PyObject* - std_unordered_map_to_py_dict(const std::unordered_map &cpp_map, - PyObject *(*KeyConvertToPy)(const K&), - PyObject *(*ValConvertToPy)(const V&) - ) { - PyObject *key = NULL; - PyObject *val = NULL; - PyObject *r = PyDict_New(); - - if (!r) { - goto except; - } - for (auto &iter: cpp_map) { - key = (*KeyConvertToPy)(iter.first); - if (! key || PyErr_Occurred()) { - goto except; - } - val = (*ValConvertToPy)(iter.second); - if (! val || PyErr_Occurred()) { - goto except; - } - if (PyDict_SetItem(r, key, val)) { - goto except; - } - } - assert(! PyErr_Occurred()); - assert(r); - goto finally; - except: - assert(PyErr_Occurred()); - // Clean up dict - if (r) { - PyDict_Clear(r); - Py_DECREF(r); - } - r = NULL; - finally: - return r; - } - -The following will convert a ``std::unordered_map`` to a Python dict ``{bytes : bytes}``: - -.. code-block:: cpp - - std::unordered_map cpp_map { - {"Foo", "Bar"} - }; - PyObject *py_dict = std_unordered_map_to_py_dict( - cpp_map, - &std_string_to_py_bytes, - &std_string_to_py_bytes - ); - if (! py_dict) { - // Handle failure... - } else { - // All good... - } - diff --git a/doc/sphinx/source/cpp_and_numpy.rst b/doc/sphinx/source/cpp_and_numpy.rst deleted file mode 100644 index dc32922..0000000 --- a/doc/sphinx/source/cpp_and_numpy.rst +++ /dev/null @@ -1,190 +0,0 @@ -.. toctree:: - :maxdepth: 2 - -.. _cpp_and_numpy: - -==================================== -C++ and the Numpy C API -==================================== - -`Numpy `_ is a powerful arrary based data structure with fast vector and array operations. It has a fully featured `C API `_. This section describes some aspects of using Numpy with C++. - ------------------------------------- -Initialising Numpy ------------------------------------- - -The Numpy C API must be setup so that a number of static data structures are initialised correctly. The way to do this is to call ``import_array()`` which makes a number of Python import statements so the Python interpreter must be initialised first. This is described in detail in the `Numpy documentation `_ so this document just presents a cookbook approach. - - ------------------------------------- -Verifying Numpy is Initialised ------------------------------------- - -``import_array()`` always returns ``NUMPY_IMPORT_ARRAY_RETVAL`` regardless of success instead we have to check the Python error status: - -.. code-block:: cpp - - #include - #include "numpy/arrayobject.h" // Include any other Numpy headers, UFuncs for example. - - // Initialise Numpy - import_array(); - if (PyErr_Occurred()) { - std::cerr << "Failed to import numpy Python module(s)." << std::endl; - return NULL; // Or some suitable return value to indicate failure. - } - -In other running code where Numpy is expected to be initialised then ``PyArray_API`` should be non-NULL and this can be asserted: - -.. code-block:: cpp - - assert(PyArray_API); - ------------------------------------- -Numpy Initialisation Techniques ------------------------------------- - - -Initialising Numpy in a CPython Module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Taking the simple example of a module from the `Python documentation `_ we can add Numpy access just by including the correct Numpy header file and calling ``import_numpy()`` in the module initialisation code: - -.. code-block:: cpp - - #include - - #include "numpy/arrayobject.h" // Include any other Numpy headers, UFuncs for example. - - static PyMethodDef SpamMethods[] = { - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods - }; - - PyMODINIT_FUNC - PyInit_spam(void) { - ... - assert(! PyErr_Occurred()); - import_numpy(); // Initialise Numpy - if (PyErr_Occurred()) { - return NULL; - } - ... - return PyModule_Create(&spammodule); - } - -That is fine for a singular translation unit but you have multiple translation units then each has to initialise the Numpy API which is a bit extravagant. The following sections describe how to manage this with multiple translation units. - -Initialising Numpy in Pure C++ Code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This is mainly for development and testing of C++ code that uses Numpy. Your code layout might look something like this where ``main.cpp`` has a ``main()`` entry point and ``class.h`` has your class declarations and ``class.cpp`` has their implementations, like this:: - - . - └── src - └── cpp -    ├── class.cpp -    ├── class.h -    └── main.cpp - -The way of managing Numpy initialisation and access is as follows. In ``class.h`` choose a unique name such as ``awesome_project`` then include: - -.. code-block:: cpp - - #define PY_ARRAY_UNIQUE_SYMBOL awesome_project_ARRAY_API - #include "numpy/arrayobject.h" - -In the implementation file ``class.cpp`` we do not want to import Numpy as that is going to be handled by ``main()`` in ``main.cpp`` so we put this at the top: - -.. code-block:: cpp - - #define NO_IMPORT_ARRAY - #include "class.h" - -Finally in ``main.cpp`` we initialise Numpy: - -.. code-block:: cpp - - #include "Python.h" - #include "class.h" - - int main(int argc, const char * argv[]) { - // ... - // Initialise the Python interpreter - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - Py_SetProgramName(program); /* optional but recommended */ - Py_Initialize(); - // Initialise Numpy - import_array(); - if (PyErr_Occurred()) { - std::cerr << "Failed to import numpy Python module(s)." << std::endl; - return -1; - } - assert(PyArray_API); - // ... - } - -If you have multiple .h, .cpp files then it might be worth having a single .h file, say ``numpy_init.h`` with just this in: - -.. code-block:: cpp - - #define PY_ARRAY_UNIQUE_SYMBOL awesome_project_ARRAY_API - #include "numpy/arrayobject.h" - -Then each implementation .cpp file has: - -.. code-block:: cpp - - #define NO_IMPORT_ARRAY - #include "numpy_init.h" - #include "class.h" // Class declarations - -And ``main.cpp`` has: - -.. code-block:: cpp - - #include "numpy_init.h" - #include "class_1.h" - #include "class_2.h" - #include "class_3.h" - - int main(int argc, const char * argv[]) { - // ... - import_array(); - if (PyErr_Occurred()) { - std::cerr << "Failed to import numpy Python module(s)." << std::endl; - return -1; - } - assert(PyArray_API); - // ... - } - -Initialising Numpy in a CPython Module using C++ Code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Supposing you have laid out your source code in the following fashion:: - - . - └── src - ├── cpp - │   ├── class.cpp - │   └── class.h - └── cpython - └── module.c - -This is a hybrid of the above and typical for CPython C++ extensions where ``module.c`` contains the CPython code that allows Python to access the pure C++ code. - -The code in ``class.h`` and ``class.cpp`` is unchanged and the code in ``module.c`` is essentially the same as that of a CPython module as described above where ``import_array()`` is called from within the ``PyInit_`` function. diff --git a/doc/sphinx/source/debugging/debug.rst b/doc/sphinx/source/debugging/debug.rst index 9248ee4..9e30d27 100644 --- a/doc/sphinx/source/debugging/debug.rst +++ b/doc/sphinx/source/debugging/debug.rst @@ -4,6 +4,9 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Debugging + ***************** Debugging ***************** @@ -21,3 +24,7 @@ This is very much work in progress. I will add to it/correct it as I develop new gcov debug_in_ide pyatexit + +.. todo:: + + Review the Debugging chapter as some of it might be obsolete. diff --git a/doc/sphinx/source/debugging/debug_in_ide.rst b/doc/sphinx/source/debugging/debug_in_ide.rst index a1b8b91..9364ffe 100644 --- a/doc/sphinx/source/debugging/debug_in_ide.rst +++ b/doc/sphinx/source/debugging/debug_in_ide.rst @@ -9,6 +9,9 @@ .. _debug-in-ide-label: +.. index:: + single: Debugging; IDEs + =============================================== Debuging Python C Extensions in an IDE =============================================== @@ -340,6 +343,9 @@ The complete code for ``py_import_call_execute.c`` is here: #endif +.. index:: + single: Debugging; Xcode + -------------------------------------------- Debugging Python C Extensions in Xcode -------------------------------------------- @@ -356,6 +362,33 @@ And you should get something like this: The full code for this is in *src/debugging/XcodeExample/PythonSubclassList/*. + +-------------------------------------------- +Using a Debug Version of Python C with Xcode +-------------------------------------------- + +To get Xcode to use a debug version of Python first build Python from source assumed here to be ```` with, as a minimum, ``--with-pydebug``. This example is using Python 3.6: + +.. code-block:: console + + cd + mkdir debug-framework + cd debug-framework/ + ../configure --with-pydebug --without-pymalloc --with-valgrind --enable-framework + make + +Then in Xcode select the project and "Add files to ..." and add: + +* ``/debug-framework/Python.framework/Versions/3.6/Python`` +* ``/debug-framework/libpython3.6d.a`` + +In "Build Settings": + +* add ``/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/`` to "Header Search Paths". Alternatively add both ``/Include`` *and* ``/debug-framework`` to "Header Search Paths", the latter is needed for ``pyconfig.h``. +* add ``/debug-framework`` to "Library Search Paths". + +Now you should be able to step into the CPython code. + -------------------------------------------- Debugging Python C Extensions in Eclipse -------------------------------------------- diff --git a/doc/sphinx/source/debugging/debug_python.rst b/doc/sphinx/source/debugging/debug_python.rst index c773379..f27a615 100644 --- a/doc/sphinx/source/debugging/debug_python.rst +++ b/doc/sphinx/source/debugging/debug_python.rst @@ -9,6 +9,9 @@ .. _debug-version-of-python-label: +.. index:: + single: Debugging; Debug Version of Python + =============================================== Building and Using a Debug Version of Python =============================================== @@ -16,19 +19,42 @@ Building and Using a Debug Version of Python There is a spectrum of debug builds of Python that you can create. This chapter describes how to create them. +.. index:: + single: Debugging; Building Python + -------------------------------------------- Building a Standard Debug Version of Python -------------------------------------------- -Download and unpack the Python source. Then in the source directory create a debug directory for the debug build: +Download check and and unpack the Python source into the directory of your choice: + +.. code-block:: bash + + $ curl -o Python-3.13.2.tgz https://www.python.org/ftp/python/3.13.2/Python-3.13.2.tgz + # Get the Gzipped source tarball md5 from: + # https://www.python.org/downloads/release/python-3132/ "6192ce4725d9c9fc0e8a1cd38410b417" + $ md5 Python-3.13.2.tgz | grep 6192ce4725d9c9fc0e8a1cd38410b417 + MD5 (Python-3.13.2.tgz) = 6192ce4725d9c9fc0e8a1cd38410b417 + # No output would be a md5 missmatch. + $ tmp echo $? + 0 + # 1 would be a md5 missmatch. + $ tar -xzf Python-3.13.2.tgz + $ cd Python-3.13.2 + + +Then in the source directory create a debug directory for the debug build: .. code-block:: bash - mkdir debug - cd debug - ../configure --with-pydebug - make - make test + $ mkdir debug + $ cd debug + $ ../configure --with-pydebug + $ make + $ make test + +.. index:: + single: Debugging; Python Build Macros ----------------------- Specifying Macros @@ -38,53 +64,69 @@ They can be specified at the configure stage, this works: .. code-block:: bash - ../configure CFLAGS='-DPy_DEBUG -DPy_TRACE_REFS' --with-pydebug - make + $ ../configure CFLAGS='-DPy_DEBUG -DPy_TRACE_REFS' --with-pydebug + $ make However the python documentation suggests the alternative way of specifying them when invoking make: .. code-block:: bash - ../configure --with-pydebug - make EXTRA_CFLAGS="-DPy_REF_DEBUG" + $ ../configure --with-pydebug + $ make EXTRA_CFLAGS="-DPy_REF_DEBUG" I don't know why one way would be regarded as better than the other. +.. index:: + pair: Debugging; Py_DEBUG + pair: Debugging; Py_REF_DEBUG + pair: Debugging; Py_TRACE_REFS + pair: Debugging; WITH_PYMALLOC + pair: Debugging; PYMALLOC_DEBUG + pair: Debugging; LLTRACE + --------------------------- The Debug Builds --------------------------- -The builds are controlled by the following macros: - -=================== ======================================================= ============== -Macro Description Must Rebuild - Extensions? -=================== ======================================================= ============== -``Py_DEBUG`` A standard debug build. ``Py_DEBUG`` sets Yes - ``LLTRACE``, ``Py_REF_DEBUG``, ``Py_TRACE_REFS``, and - ``PYMALLOC_DEBUG`` (if ``WITH_PYMALLOC`` is enabled). -``Py_REF_DEBUG`` Turn on aggregate reference counting which will be No - displayed in the interactive interpreter when - invoked with ``-X showrefcount`` on the command line. - If you are not keeping references to objects and the - count is increasing there is probably a leak. - Also adds ``sys.gettotalrefcount()`` to the ``sys`` - module and this returns the total number of references. -``Py_TRACE_REFS`` Turns on reference tracing. Yes - Sets ``Py_REF_DEBUG``. -``COUNT_ALLOCS`` Keeps track of the number of objects of each type have Yes - been allocated and how many freed. - See: :ref:`debug-version-of-python-COUNT_ALLOCS-label` -``WITH_PYMALLOC`` Enables Pythons small memory allocator. For Valgrind No - this must be disabled, if using Pythons malloc - debugger (using ``PYMALLOC_DEBUG``) this must be - enabled. - See: :ref:`debug-version-of-python-memory_alloc-label` -``PYMALLOC_DEBUG`` Enables Python's malloc debugger that annotates No - memory blocks. Requires ``WITH_PYMALLOC``. - See: :ref:`debug-version-of-python-memory_alloc-label` -=================== ======================================================= ============== +The builds are controlled by the following macros. +The third column shows if CPython C extensions have to be rebuilt against that version of Python: + +.. list-table:: Debug Macros + :widths: 20 70 10 + :header-rows: 1 + + * - Macro + - Description + - Rebuild EXT? + * - ``Py_DEBUG`` + - A standard debug build. ``Py_DEBUG`` sets ``LLTRACE``, ``Py_REF_DEBUG``, ``Py_TRACE_REFS``, and + ``PYMALLOC_DEBUG`` (if ``WITH_PYMALLOC`` is enabled). + - Yes + * - ``Py_REF_DEBUG`` + - Turn on aggregate reference counting which will be + displayed in the interactive interpreter when + invoked with ``-X showrefcount`` on the command line. + If you are not keeping references to objects and the + count is increasing there is probably a leak. + Also adds ``sys.gettotalrefcount()`` to the ``sys`` + module and this returns the total number of references. + - No + * - ``Py_TRACE_REFS`` + - Turns on reference tracing. Sets ``Py_REF_DEBUG``. + - Yes + * - ``WITH_PYMALLOC`` + - Enables Pythons small memory allocator. For Valgrind + this must be disabled, if using Pythons malloc + debugger (using ``PYMALLOC_DEBUG``) this must be + enabled. + See: :ref:`debug-version-of-python-memory_alloc-label` + - No + * - ``PYMALLOC_DEBUG`` + - Enables Python's malloc debugger that annotates + memory blocks. Requires ``WITH_PYMALLOC``. + See: :ref:`debug-version-of-python-memory_alloc-label` + - No Here is the description of other debug macros that are set by one of the macros above: @@ -94,19 +136,12 @@ Macro Description ``LLTRACE`` Low level tracing. See ``Python/ceval.c``. =================== ======================================================= -In the source directory: - -.. code-block:: bash - - mkdir debug - cd debug - ../configure --with-pydebug - make - make test - .. _debug-version-of-python-memory_alloc-label: +.. index:: + single: Debugging; Python's Memory Allocator + --------------------------- Python's Memory Allocator --------------------------- @@ -122,8 +157,8 @@ To make a version of Python with its memory allocator suitable for use with Valg .. code-block:: bash - ../configure --with-pydebug --without-pymalloc - make + $ ../configure --with-pydebug --without-pymalloc + $ make See :ref:`using-valgrind-label` for using Valgrind. @@ -131,181 +166,185 @@ To make a version of Python with its memory allocator using Python's malloc debu .. code-block:: bash - ../configure CFLAGS='-DPYMALLOC_DEBUG' --with-pydebug - make + $ ../configure CFLAGS='-DPYMALLOC_DEBUG' --with-pydebug + $ make Or: .. code-block:: bash - ../configure --with-pydebug - make EXTRA_CFLAGS="-DPYMALLOC_DEBUG" + $ ../configure --with-pydebug + $ make EXTRA_CFLAGS="-DPYMALLOC_DEBUG" This builds Python with the ``WITH_PYMALLOC`` and ``PYMALLOC_DEBUG`` macros defined. +.. index:: + pair: Debugging; Access After Free + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Finding Access after Free With ``PYMALLOC_DEBUG`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python built with ``PYMALLOC_DEBUG`` is the most effective way of detecting access after free. For example if we have this CPython code: +TODO: + +Python built with ``PYMALLOC_DEBUG`` is the most effective way of detecting access after free. +For example if we have this CPython code: .. code-block:: c - static PyObject *access_after_free(PyObject *pModule) { - PyObject *pA = PyLong_FromLong(1024L); + static PyObject *access_after_free(PyObject *Py_UNUSED(module)) { + PyObject *pA = PyLong_FromLong(1024L * 1024L); + fprintf( + stdout, + "%s(): Before Py_DECREF(0x%p) Ref count: %zd\n", + __FUNCTION__, (void *)pA, Py_REFCNT(pA) + ); + PyObject_Print(pA, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + Py_DECREF(pA); - PyObject_Print(pA, stdout, 0); + + fprintf( + stdout, + "%s(): After Py_DECREF(0x%p) Ref count: %zd\n", + __FUNCTION__, (void *)pA, Py_REFCNT(pA) + ); + PyObject_Print(pA, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + Py_RETURN_NONE; } -And we call this from the interpreter we get a diagnostic: +And we call this from the interpreter what we get is undefined and may vary from Python version to version. +Here is Python 3.9: .. code-block:: python - Python 3.4.3 (default, Sep 16 2015, 16:56:10) - [GCC 4.2.1 Compatible Apple LLVM 6.0 (clang-600.0.51)] on darwin - Type "help", "copyright", "credits" or "license" for more information. - >>> import cPyRefs - >>> cPyRefs.afterFree() - - >>> - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Getting Statistics on PyMalloc -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If the environment variable ``PYTHONMALLOCSTATS`` exists when running Python built with ``WITH_PYMALLOC``+``PYMALLOC_DEBUG`` then a (detailed) report of pymalloc activity is output on stderr whenever a new 'arena' is allocated. + # $ python + # Python 3.9.7 (v3.9.7:1016ef3790, Aug 30 2021, 16:39:15) + # [Clang 6.0 (clang-600.0.57)] on darwin + # Type "help", "copyright", "credits" or "license" for more information. + >>> from cPyExtPatt import cPyRefs + >>> cPyRefs.access_after_free() + access_after_free(): Before Py_DECREF(0x0x7fe55f1e2fb0) Ref count: 1 + 1048576 + access_after_free(): After Py_DECREF(0x0x7fe55f1e2fb0) Ref count: 140623120051984 + 1048576 + >>> -.. code-block:: bash - - PYTHONMALLOCSTATS=1 python.exe - -I have no special knowledge about the output you see when running Python this way which looks like this:: +And Python 3.13: - >>> cPyRefs.leakNewRefs(1000, 10000) - loose_new_reference: value=1000 count=10000 - Small block threshold = 512, in 64 size classes. - - class size num pools blocks in use avail blocks - ----- ---- --------- ------------- ------------ - 4 40 2 139 63 - 5 48 1 2 82 - ... - 62 504 3 21 3 - 63 512 3 18 3 - - # times object malloc called = 2,042,125 - # arenas allocated total = 636 - # arenas reclaimed = 1 - # arenas highwater mark = 635 - # arenas allocated current = 635 - 635 arenas * 262144 bytes/arena = 166,461,440 - - # bytes in allocated blocks = 162,432,624 - # bytes in available blocks = 116,824 - 0 unused pools * 4096 bytes = 0 - # bytes lost to pool headers = 1,950,720 - # bytes lost to quantization = 1,961,272 - # bytes lost to arena alignment = 0 - Total = 166,461,440 - Small block threshold = 512, in 64 size classes. - - class size num pools blocks in use avail blocks - ----- ---- --------- ------------- ------------ - 4 40 2 139 63 - 5 48 1 2 82 - ... - 62 504 3 21 3 - 63 512 3 18 3 - - # times object malloc called = 2,045,325 - # arenas allocated total = 637 - # arenas reclaimed = 1 - # arenas highwater mark = 636 - # arenas allocated current = 636 - 636 arenas * 262144 bytes/arena = 166,723,584 - - # bytes in allocated blocks = 162,688,624 - # bytes in available blocks = 116,824 - 0 unused pools * 4096 bytes = 0 - # bytes lost to pool headers = 1,953,792 - # bytes lost to quantization = 1,964,344 - # bytes lost to arena alignment = 0 - Total = 166,723,584 - Small block threshold = 512, in 64 size classes. - - class size num pools blocks in use avail blocks - ----- ---- --------- ------------- ------------ - 4 40 2 139 63 - 5 48 1 2 82 - ... - 62 504 3 21 3 - 63 512 3 18 3 - - # times object malloc called = 2,048,525 - # arenas allocated total = 638 - # arenas reclaimed = 1 - # arenas highwater mark = 637 - # arenas allocated current = 637 - 637 arenas * 262144 bytes/arena = 166,985,728 - - # bytes in allocated blocks = 162,944,624 - # bytes in available blocks = 116,824 - 0 unused pools * 4096 bytes = 0 - # bytes lost to pool headers = 1,956,864 - # bytes lost to quantization = 1,967,416 - # bytes lost to arena alignment = 0 - Total = 166,985,728 - loose_new_reference: DONE - - -.. _debug-version-of-python-COUNT_ALLOCS-label: +.. code-block:: python ------------------------------------------------ -Python Debug build with ``COUNT_ALLOCS`` ------------------------------------------------ + # $ python + # Python 3.13.1 (v3.13.1:06714517797, Dec 3 2024, 14:00:22) [Clang 15.0.0 (clang-1500.3.9.4)] on darwin + # Type "help", "copyright", "credits" or "license" for more information. + >>> from cPyExtPatt import cPyRefs + >>> cPyRefs.access_after_free() + access_after_free(): Before Py_DECREF(0x0x102465890) Ref count: 1 + 1048576 + access_after_free(): After Py_DECREF(0x0x102465890) Ref count: 4333131856 + 0 + >>> -A Python debug build with ``COUNT_ALLOCS`` give some additional information about each object *type* (not the individual objects themselves). A ``PyObject`` grows some extra fields that track the reference counts for that type. The fields are: +And we call this from the (debug) Python 3.13 interpreter we get a diagnostic: -=============== ==================================================================== -Field Description -=============== ==================================================================== -``tp_allocs`` The number of times an object of this type was allocated. -``tp_frees`` The number of times an object of this type was freed. -``tp_maxalloc`` The maximum seen value of ``tp_allocs - tp_frees`` so this is the - maximum count of this type allocated at the same time. -=============== ==================================================================== +.. code-block:: python -The ``sys`` module also gets an extra function ``sys.getcounts()`` that returns a list of tuples: ``[(tp_typename, tp_allocs, tp_frees, tp_maxalloc), ...]``. + # (PyExtPatt_3.13.2_Debug) PythonExtensionPatterns git:(develop) $ python + # Python 3.13.2 (main, Mar 9 2025, 11:01:02) [Clang 14.0.3 (clang-1403.0.22.14.1)] on darwin + # Type "help", "copyright", "credits" or "license" for more information. + >>> from cPyExtPatt import cPyRefs + >>> cPyRefs.access_after_free() + access_after_free(): Before Py_DECREF(0x0x10a984e00) Ref count: 1 + 1048576 + access_after_free(): After Py_DECREF(0x0x10a984e00) Ref count: -2459565876494606883 + + >>> +.. index:: + single: Debugging; PyMalloc Statistics -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Building the Python Executable with ``COUNT_ALLOCS`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Getting Statistics on PyMalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Either: +If the environment variable ``PYTHONMALLOCSTATS`` exists when running Python built with +``WITH_PYMALLOC``+``PYMALLOC_DEBUG`` then a (detailed) report of pymalloc activity is output on stderr whenever +a new 'arena' is allocated. .. code-block:: bash - ../configure CFLAGS='-DCOUNT_ALLOCS' --with-pydebug - make + $ PYTHONMALLOCSTATS=1 python -Or: - -.. code-block:: bash +I have no special knowledge about the output you see when running Python this way which looks like this:: - ../configure --with-pydebug - make EXTRA_CFLAGS="-DCOUNT_ALLOCS" + >>> from cPyExtPatt import cPyRefs + >>> cPyRefs.leak_new_reference(1000, 10000) + loose_new_reference: value=1000 count=10000 + Small block threshold = 512, in 32 size classes. -.. warning:: + class size num pools blocks in use avail blocks + ----- ---- --------- ------------- ------------ + 1 32 1 83 427 + 2 48 1 146 194 + 3 64 48 12240 0 + 4 80 82 16690 38 + 5 96 74 12409 171 + 6 112 22 3099 91 + 7 128 10 1156 114 + 8 144 12 1346 10 + 9 160 4 389 19 + 10 176 4 366 2 + 11 192 32 2649 71 + 12 208 3 173 61 + 13 224 2 132 12 + 14 240 13 860 24 + 15 256 5 281 34 + 16 272 6 344 16 + 17 288 7 348 44 + 18 304 8 403 21 + 19 320 4 198 6 + 20 336 4 183 9 + 21 352 5 198 32 + 22 368 3 119 13 + 23 384 3 104 22 + 24 400 3 106 14 + 25 416 3 98 19 + 26 432 8 295 1 + 27 448 2 64 8 + 28 464 3 70 35 + 29 480 2 44 24 + 30 496 2 59 5 + 31 512 2 45 17 + + # arenas allocated total = 6 + # arenas reclaimed = 0 + # arenas highwater mark = 6 + # arenas allocated current = 6 + 6 arenas * 1048576 bytes/arena = 6,291,456 + + # bytes in allocated blocks = 5,927,280 + # bytes in available blocks = 224,832 + 0 unused pools * 16384 bytes = 0 + # bytes lost to pool headers = 18,144 + # bytes lost to quantization = 22,896 + # bytes lost to arena alignment = 98,304 + Total = 6,291,456 + + arena map counts + # arena map mid nodes = 1 + # arena map bot nodes = 1 + + # bytes lost to arena map root = 262,144 + # bytes lost to arena map mid = 262,144 + # bytes lost to arena map bot = 131,072 + Total = 655,360 + loose_new_reference: DONE - When using ``COUNT_ALLOCS`` any Python extensions now need to be rebuilt with this Python executable as it fundementally changes the structure of a ``PyObject``. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using the Python Executable with ``COUNT_ALLOCS`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -An example of using this build is here: :ref:`leaked-new-references-usingCOUNT_ALLOCS-label` +.. index:: + pair: Debugging; sysconfig ----------------------------------------------------------- Identifying the Python Build Configuration from the Runtime diff --git a/doc/sphinx/source/debugging/debug_tactics.rst b/doc/sphinx/source/debugging/debug_tactics.rst index d754d5a..5b513bf 100644 --- a/doc/sphinx/source/debugging/debug_tactics.rst +++ b/doc/sphinx/source/debugging/debug_tactics.rst @@ -7,6 +7,8 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Debugging; Tactics ================================= Debugging Tactics @@ -15,6 +17,10 @@ Debugging Tactics So what is the problem that you are trying to solve? +.. index:: + single: Access After Free + single: Debugging; Access After Free + ---------------------------- Access After Free ---------------------------- @@ -45,82 +51,7 @@ Solution Python 3.4.3 (default, Sep 16 2015, 16:56:10) [GCC 4.2.1 Compatible Apple LLVM 6.0 (clang-600.0.51)] on darwin Type "help", "copyright", "credits" or "license" for more information. - >>> import cPyRefs + >>> from cPyExtPatt import cPyRefs >>> cPyRefs.afterFree() >>> - ----------------------------------- -``Py_INCREF`` called too often ----------------------------------- - -**Summary:** If ``Py_INCREF`` is called once or more too often then memory will be held onto despite those objects not being visible in ``globals()`` or ``locals()``. - -**Symptoms:** You run a test that creates objects and uses them. As the test exits all the objects should go out of scope and be de-alloc'd however you observe that memory is being permanently lost. - -^^^^^^^^^^^^^^^^^^^ -Problem -^^^^^^^^^^^^^^^^^^^ - -We can create a simulation of this by creating two classes, with one we will create a leak caused by an excessive ``Py_INCREF`` (we do this by calling ``cPyRefs.incref()``). The other class instance will not be leaked. - -Here is the code for incrementing the reference count in the ``cPyRefs`` module: - -.. code-block:: c - - /* Just increfs a PyObject. */ - static PyObject *incref(PyObject *pModule, PyObject *pObj) { - fprintf(stdout, "incref(): Ref count was: %zd\n", pObj->ob_refcnt); - Py_INCREF(pObj); - fprintf(stdout, "incref(): Ref count now: %zd\n", pObj->ob_refcnt); - Py_RETURN_NONE; - } - -And the Python interpreter session we create two instances and excessively incref one of them: - -.. code-block:: python - - >>> import cPyRefs # So we can create a leak - >>> class Foo : pass # Foo objects will be leaked - ... - >>> class Bar : pass # Bar objects will not be leaked - ... - >>> def test_foo_bar(): - ... f = Foo() - ... b = Bar() - ... # Now increment the reference count of f, but not b - ... # This simulates what might happen in a leaky extension - ... cPyRefs.incref(f) - ... - >>> # Call the test, the output comes from cPyRefs.incref() - >>> test_foo_bar() - incref(): Ref count was: 2 - incref(): Ref count now: 3 - - -^^^^^^^^^^^^^^^^^^^ -Solution -^^^^^^^^^^^^^^^^^^^ - -Use a debug version of Python with ``COUNT_ALLOCS`` defined. This creates ``sys.getcounts()`` which lists, for each type, how many allocs and de-allocs have been made. Notice the difference between ``Foo`` and ``Bar``: - -.. code-block:: python - - >>> import sys - >>> sys.getcounts() - [ - ('Bar', 1, 1, 1), - ('Foo', 1, 0, 1), - ... - ] - -This should focus your attention on the leaky type ``Foo``. - -You can find the count of all live objects by doing this: - -.. code-block:: python - - >>> still_live = [(v[0], v[1] - v[2]) for v in sys.getcounts() if v[1] > v[2]] - >>> still_live - [('Foo', 1), ... ('str', 7783), ('dict', 714), ('tuple', 3875)] - diff --git a/doc/sphinx/source/debugging/debug_tools.rst b/doc/sphinx/source/debugging/debug_tools.rst index 8d504f2..339872f 100644 --- a/doc/sphinx/source/debugging/debug_tools.rst +++ b/doc/sphinx/source/debugging/debug_tools.rst @@ -7,6 +7,8 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Debugging; Tools ================================= Debugging Tools @@ -16,10 +18,14 @@ First create your toolbox, in this one we have: * Debug version of Python - great for finding out more detail of your Python code as it executes. * Valgrind - the goto tool for memory leaks. It is a little tricky to get working but should be in every developers toolbox. -* OS memory monitioring - this is a quick and simple way of identifying whether memory leaks are happening or not. An example is given below: :ref:`simple-memory-monitor-label` +* OS memory monitioring - this is a quick and simple way of identifying whether memory leaks are happening or not. + An example is given below: :ref:`simple-memory-monitor-label` .. _debug-tools-debug-python-label: +.. index:: + single: Debugging; A Debug Python Version + ------------------------------------------------ Build a Debug Version of Python ------------------------------------------------ @@ -43,6 +49,9 @@ See here :ref:`debug-version-of-python-label` for instructions on how to do this .. _debug-tools-valgrind-label: +.. index:: + single: Debugging; Valgrind + ------------------------------------------------ Valgrind ------------------------------------------------ @@ -56,11 +65,20 @@ Here :ref:`leaked-new-references-valgrind-label` is an example of finding a leak .. _simple-memory-monitor-label: +.. index:: + single: Debugging; Memory Monitor + single: Memory Monitor + see: Memory Monitor; pymemtrace + see: pymemtrace; Memory Monitor + ------------------------------------------------ A Simple Memory Monitor ------------------------------------------------ -Here is a simple process memory monitor using the ``psutil`` library: +A useful technique is to monitor the memory usage of a Python program. +Here is a simple process memory monitor using the ``psutil`` library. +See the :ref:`memory_leaks-label` chapter for a more comprehensive approach, in particular +:ref:`memory-leaks.pymemtrace`. .. code-block:: python diff --git a/doc/sphinx/source/debugging/gcov.rst b/doc/sphinx/source/debugging/gcov.rst index 1efb4f6..8970e11 100644 --- a/doc/sphinx/source/debugging/gcov.rst +++ b/doc/sphinx/source/debugging/gcov.rst @@ -9,6 +9,9 @@ .. _gcov-label: +.. index:: + single: Debugging; gcov + =============================================== Using gcov for C/C++ Code Coverage =============================================== diff --git a/doc/sphinx/source/debugging/leak_newrefs_vg.rst b/doc/sphinx/source/debugging/leak_newrefs_vg.rst index ae35f17..2e41339 100644 --- a/doc/sphinx/source/debugging/leak_newrefs_vg.rst +++ b/doc/sphinx/source/debugging/leak_newrefs_vg.rst @@ -10,6 +10,10 @@ .. _leaked-new-references-label: +.. index:: + single: References; Finding Leaks + single: Reference Counts; Finding Leaks + =============================================== Leaked New References =============================================== @@ -65,12 +69,24 @@ And we add this to the ``cPyRefs`` module function table as the python function {NULL, NULL, 0, NULL} /* Sentinel */ }; -In Python first we check what the size of a long is then we call the leaky function with the value 1000 (not the values -5 to 255 which are interned) one million times and there should be a leak of one million times the size of a long:: +In Python first we check what the size of a long is then we call the leaky function with the value 1000 +(not the values -5 to 255 which are interned) one million times and there should be a leak of one million times +the size of a long: + +This is with a debug version of Python 3.13: + +.. code-block:: bash + + $ python + Python 3.13.2 (main, Mar 9 2025, 13:27:38) [Clang 15.0.0 (clang-1500.0.40.1)] on darwin + Type "help", "copyright", "credits" or "license" for more information. + +.. code-block:: python >>> import sys >>> sys.getsizeof(1000) 44 - >>> import cPyRefs + >>> from cPyExtPatt import cPyRefs >>> cPyRefs.leakNewRefs(1000, 1000000) loose_new_reference: value=1000 count=1000000 loose_new_reference: DONE @@ -78,6 +94,10 @@ In Python first we check what the size of a long is then we call the leaky funct This should generate a leak of 44Mb or thereabouts. +.. index:: + single: References; Recognising Leaks + single: Reference Counts; Recognising Leaks + ---------------------------------- Recognising Leaked New References ---------------------------------- @@ -114,7 +134,7 @@ In a second shell fire up pidmon.py with this PID: Go back to the first shell and import ``cPyRefs``:: - >>> import cPyRefs + >>> from cPyExtPatt import cPyRefs >>> cPyRefs.leakNewRefs(1000, 1000000) loose_new_reference: value=1000 count=1000000 loose_new_reference: DONE @@ -151,7 +171,7 @@ For example: .. code-block:: python >>> import sys - >>> import cPyRefs + >>> from cPyExtPatt import cPyRefs >>> dir() ['__builtins__', '__doc__', '__loader__', '__name__', '__package__', '__spec__', 'cPyRefs', 'sys'] >>> sys.gettotalrefcount() @@ -175,66 +195,13 @@ And those references are not collectable:: 1055519 -.. _leaked-new-references-usingCOUNT_ALLOCS-label: - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Observing the Reference Counts for a Particular Type -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you have a debug build with ``COUNT_ALLOCS`` [See: :ref:`debug-version-of-python-COUNT_ALLOCS-label`] defined you can see the references counts for each type. This build will have a new function ``sys.getcounts()`` which returns a list of tuples ``(tp_name, tp_allocs, tp_frees, tp_maxalloc)`` where ``tp_maxalloc`` is the maximum ever seen value of the reference ``tp_allocs - tp_frees``. The list is ordered by time of first object allocation: - -.. code-block:: python - - >>> import pprint - >>> import sys - >>> pprint.pprint(sys.getcounts()) - [('Repr', 1, 0, 1), - ('symtable entry', 3, 3, 1), - ('OSError', 1, 1, 1), - ... - ('int', 3342, 2630, 712), - ... - ('dict', 1421, 714, 714), - ('tuple', 13379, 9633, 3746)] - -We can try our leaky code: - -.. code-block:: python - - >>> import cPyRefs - >>> cPyRefs.leakNewRefs(1000, 1000000) - loose_new_reference: value=1000 count=1000000 - loose_new_reference: DONE - >>> pprint.pprint(sys.getcounts()) - [('memoryview', 103, 103, 1), - ... - ('int', 1004362, 3650, 1000712), - ... - ('dict', 1564, 853, 718), - ('tuple', 22986, 19236, 3750)] - -There is a big jump in ``tp_maxalloc`` for ints that is worth investigating. - -When the Python process finishes you get a dump of this list as the interpreter is broken down:: - -.. code-block:: console - - memoryview alloc'd: 210, freed: 210, max in use: 1 - managedbuffer alloc'd: 210, freed: 210, max in use: 1 - PrettyPrinter alloc'd: 2, freed: 2, max in use: 1 - ... - int alloc'd: 1005400, freed: 4887, max in use: 1000737 - ... - str alloc'd: 21920, freed: 19019, max in use: 7768 - dict alloc'd: 1675, freed: 1300, max in use: 718 - tuple alloc'd: 32731, freed: 31347, max in use: 3754 - fast tuple allocs: 28810, empty: 2101 - fast int allocs: pos: 7182, neg: 20 - null strings: 69, 1-strings: 5 - .. _leaked-new-references-valgrind-label: +.. index:: + single: References; Valgrind + single: Reference Counts; Valgrind + ------------------------------------------ Finding Where the Leak is With Valgrind ------------------------------------------ @@ -251,7 +218,7 @@ Lets run our debug version of Python with Valgrind and see if we can spot the le Then run this code:: - >>> import cPyRefs + >>> from cPyExtPatt import cPyRefs >>> cPyRefs.leakNewRefs(1000, 1000000) loose_new_reference: value=1000 count=1000000 loose_new_reference: DONE diff --git a/doc/sphinx/source/debugging/pyatexit.rst b/doc/sphinx/source/debugging/pyatexit.rst index d4bd0d0..ac20904 100644 --- a/doc/sphinx/source/debugging/pyatexit.rst +++ b/doc/sphinx/source/debugging/pyatexit.rst @@ -7,13 +7,22 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Debugging; pyatexit + ======================================================= Instrumenting the Python Process for Your Structures ======================================================= -Some debugging problems can be solved by instrumenting your C extensions for the duration of the Python process and reporting what happened when the process terminates. The data could be: the number of times classes were instantiated, functions called, memory allocations/deallocations or anything else that you wish. +Some debugging problems can be solved by instrumenting your C extensions for the duration of the Python process and +reporting what happened when the process terminates. +The data could be: the number of times classes were instantiated, functions called, memory allocations/deallocations +or anything else that you wish. -To take a simple case, suppose we have a class that implements a up/down counter and we want to count how often each ``inc()`` and ``dec()`` function is called during the entirety of the Python process. We will create a C extension that has a class that has a single member (an interger) and two functions that increment or decrement that number. If it was in Python it would look like this: +To take a simple case, suppose we have a class that implements a up/down counter and we want to count how often each +``inc()`` and ``dec()`` function is called during the entirety of the Python process. +We will create a C extension that has a class that has a single member (an interger) and two functions that increment +or decrement that number. If it was in Python it would look like this: .. code-block:: python @@ -27,15 +36,19 @@ To take a simple case, suppose we have a class that implements a up/down counter def dec(self): self.count -= 1 -What we would like to do is to count how many times ``inc()`` and ``dec()`` are called on *all* instances of these objects and summarise them when the Python process exits [#f1]_. +What we would like to do is to count how many times ``inc()`` and ``dec()`` are called on *all* instances of these +objects and summarise them when the Python process exits [#f1]_. -There is an interpreter hook ``Py_AtExit()`` that allows you to register C functions that will be executed as the Python interpreter exits. This allows you to dump information that you have gathered about your code execution. +There is an interpreter hook ``Py_AtExit()`` that allows you to register C functions that will be executed as the +Python interpreter exits. +This allows you to dump information that you have gathered about your code execution. ------------------------------------------- An Implementation of a Counter ------------------------------------------- -First here is the module ``pyatexit`` with the class ``pyatexit.Counter`` with no intrumentation (it is equivelent to the Python code above). We will add the instrumentation later: +First here is the module ``pyatexit`` with the class ``pyatexit.Counter`` with no intrumentation (it is equivalent to +the Python code above). We will add the instrumentation later: .. code-block:: c @@ -187,13 +200,15 @@ Building this with ``python3 setup.py build_ext --inplace`` we can check everyth Instrumenting the Counter ------------------------------------------- -To add the instrumentation we will declare a macro ``COUNT_ALL_DEC_INC`` to control whether the compilation includes instrumentation. +To add the instrumentation we will declare a macro ``COUNT_ALL_DEC_INC`` to control whether the compilation includes +instrumentation. .. code-block:: c #define COUNT_ALL_DEC_INC -In the global area of the file declare some global counters and a function to write them out on exit. This must be a ``void`` function taking no arguments: +In the global area of the file declare some global counters and a function to write them out on exit. +This must be a ``void`` function taking no arguments: .. code-block:: c @@ -210,7 +225,8 @@ In the global area of the file declare some global counters and a function to wr } #endif -In the ``Py_Counter_new`` function we add some code to register this function. This must be only done once so we use the static ``has_registered_exit_function`` to guard this: +In the ``Py_Counter_new`` function we add some code to register this function. +This must be only done once so we use the static ``has_registered_exit_function`` to guard this: .. code-block:: c @@ -234,10 +250,12 @@ In the ``Py_Counter_new`` function we add some code to register this function. T } .. note:: - ``Py_AtExit`` can take, at most, 32 functions. If the function can not be registered then ``Py_AtExit`` will return -1. + ``Py_AtExit`` can take, at most, 32 functions. If the function can not be registered then ``Py_AtExit`` will + return -1. .. warning:: - Since Python’s internal finalization will have completed before the cleanup function, no Python APIs should be called by any registered function. + Since Python’s internal finalization will have completed before the cleanup function, no Python APIs should be + called by any registered function. Now we modify the ``inc()`` and ``dec()`` functions thus: @@ -284,4 +302,5 @@ Now when we build this extension and run it we see the following: .. rubric:: Footnotes -.. [#f1] The ``atexit`` module in Python can be used to similar effect however registered functions are called at a different stage of interpreted teardown than ``Py_AtExit``. +.. [#f1] The ``atexit`` module in Python can be used to similar effect however registered functions are called at a + different stage of interpreted teardown than ``Py_AtExit``. diff --git a/doc/sphinx/source/debugging/valgrind.rst b/doc/sphinx/source/debugging/valgrind.rst index d297f05..3aea3a8 100644 --- a/doc/sphinx/source/debugging/valgrind.rst +++ b/doc/sphinx/source/debugging/valgrind.rst @@ -10,6 +10,9 @@ .. _valgrind-label: +.. index:: + single: Valgrind + =============================================== Valgrind =============================================== @@ -21,6 +24,9 @@ This is about how to build Valgrind, a Valgrind friendly version of Python and f These instructions have been tested on Mac OS X 10.9 (Mavericks). They may or may not work on other OS's +.. index:: + single: Valgrind; Building + --------------------------------- Building Valgrind --------------------------------- @@ -38,6 +44,9 @@ This should be fairly straightforward: .. _building-python-for-valgrind-label: +.. index:: + single: Valgrind; Building Python For + --------------------------------- Building Python for Valgrind --------------------------------- @@ -102,6 +111,9 @@ Check debug build .. _using-valgrind-label: +.. index:: + single: Valgrind; Using + --------------------------------- Using Valgrind --------------------------------- @@ -113,7 +125,7 @@ In the ``/Misc`` directory there is a ``valgrind-python.supp`` fi cp /Misc/valgrind-python.supp ~/valgrind-python.supp vi ~/valgrind-python.supp -Uncomment ``PyObject_Free`` and ``PyObject_Realloc`` in the valgring suppression file. +Uncomment ``PyObject_Free`` and ``PyObject_Realloc`` in the valgrind suppression file. Invoking the Python interpreter with Valgrind: diff --git a/doc/sphinx/source/exceptions.rst b/doc/sphinx/source/exceptions.rst index 1cf6cf9..604f757 100644 --- a/doc/sphinx/source/exceptions.rst +++ b/doc/sphinx/source/exceptions.rst @@ -4,25 +4,37 @@ .. toctree:: :maxdepth: 3 +.. index:: + single: Exceptions; General + ================================= Exception Raising ================================= -A brief interlude on how to communicate error conditions from C code to Python. +This is a brief interlude on how to communicate error conditions from C code to Python. +The example code for this is at ``src/cpy/cExceptions.c`` and the test examples are in +``tests/unit/test_c_exceptions.py``. These CPython calls are the most useful: * ``PyErr_SetString(...)`` - To set an exception type with a fixed string. + `PyErr_SetString() documentation `_ * ``PyErr_Format(...)`` - To set an exception type with a formatted string. + `PyErr_Format() documentation `_ * ``PyErr_Occurred()`` - To check if an exception has already been set in the flow of control. + This returns the current exception or NULL if nothing set. + `PyErr_Occurred() documentation `_ * ``PyErr_Clear()`` - Clearing any set exceptions, have good reason to do this! + `PyErr_Clear() documentation `_ +* ``PyErr_Print()`` - Print a representation of the current exception then clear any set exceptions. + `PyErr_Print() documentation `_ -Indicating an error condition is a two stage process; your code must register an exception and then indicate failure by returning ``NULL``. Here is a C function doing just that: +Indicating an error condition is a two stage process; your code must register an exception and then indicate failure +by returning ``NULL``. Here is a C function doing just that: .. code-block:: c - static PyObject *_raise_error(PyObject *module) { - + static PyObject *raise_error(PyObject *module) { PyErr_SetString(PyExc_ValueError, "Ooops."); return NULL; } @@ -31,8 +43,7 @@ You might want some dynamic information in the exception object, in that case `` .. code-block:: c - static PyObject *_raise_error_formatted(PyObject *module) { - + static PyObject *raise_error_formatted(PyObject *module) { PyErr_Format(PyExc_ValueError, "Can not read %d bytes when offset %d in byte length %d.", \ 12, 25, 32 @@ -40,53 +51,65 @@ You might want some dynamic information in the exception object, in that case `` return NULL; } -If one of the two actions is missing then the exception will not be raised correctly. For example returning ``NULL`` without setting an exception type: +If one of the two actions is missing then the exception will not be raised correctly. +For example returning ``NULL`` without setting an exception type will raise a ``SystemError``. +For example: .. code-block:: c /* Illustrate returning NULL but not setting an exception. */ - static PyObject *_raise_error_bad(PyObject *module) { + static PyObject *raise_error_bad(PyObject *module) { return NULL; } -Executing this from Python will produce a clear error message (the C function ``_raise_error_bad()`` is mapped to the Python function ``cExcep.raiseErrBad()`` :: +Executing this from Python will produce a clear error message:: - >>> cExcep.raiseErrBad() + >>> cException.raise_error_bad() Traceback (most recent call last): File "", line 1, in SystemError: error return without exception set -If the opposite error is made, that is setting an exception but not signalling then the function will succeed but leave a later runtime error: +If the opposite error is made, that is setting an exception but not signalling then the function will succeed but leave +a later runtime error: .. code-block:: c - static PyObject *_raise_error_mixup(PyObject *module) { - PyErr_SetString(PyExc_ValueError, "ERROR: _raise_error_mixup()"); + static PyObject *raise_error_silent(PyObject *module) { + PyErr_SetString(PyExc_ValueError, "ERROR: raise_error_mixup()"); Py_RETURN_NONE; } -The confusion can arise is that if a subsequent function then tests to see if an exception is set, if so signal it. It will appear that the error is coming from the second function when actually it is from the first: +The confusion can arise is that if a subsequent function then tests to see if an exception is set, if so signal it. +It will appear to the Python interpreter that the error is coming from the second function when actually it is from +the first: .. code-block:: c - static PyObject *_raise_error_mixup_test(PyObject *module) { + static PyObject *raise_error_silent_test(PyObject *module) { if (PyErr_Occurred()) { return NULL; } Py_RETURN_NONE; } +A common defensive pattern to use in functions to check an exception has been set by another function but not signalled +is to use ``assert(!PyErr_Occurred());`` at the begining of the function. + The other thing to note is that if there are multiple calls to ``PyErr_SetString`` only the last one counts: .. code-block:: c - static PyObject *_raise_error_overwrite(PyObject *module) { + static PyObject *raise_error_overwrite(PyObject *module) { + /* This will be ignored. */ PyErr_SetString(PyExc_RuntimeError, "FORGOTTEN."); - PyErr_SetString(PyExc_ValueError, "ERROR: _raise_error_overwrite()"); + PyErr_SetString(PyExc_ValueError, "ERROR: raise_error_overwrite()"); assert(PyErr_Occurred()); return NULL; } +.. index:: + single: Exceptions; Common Exception Patterns + --------------------------------- Common Exception Patterns --------------------------------- @@ -97,7 +120,8 @@ Here are some common use cases for raising exceptions. Type Checking ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A common requirement is to check the types of the arguments and raise a ``TypeError`` if they are wrong. Here is an example where we require a ``bytes`` object: +A common requirement is to check the types of the arguments and raise a ``TypeError`` if they are wrong. +Here is an example where we require a ``bytes`` object: .. code-block:: c :linenos: @@ -110,136 +134,197 @@ A common requirement is to check the types of the arguments and raise a ``TypeEr PyErr_Format(PyExc_TypeError, "Argument \"value\" to %s must be a bytes object not a \"%s\"", __FUNCTION__, Py_TYPE(arg)->tp_name); - goto except; + return NULL; } /* ... */ } -Thats fine if you have a macro such as ``PyBytes_Check`` and for your own types you can create a couple of suitable macros: +That's fine if you have a macro such as ``PyBytes_Check`` and for your own types you can create a couple of suitable +macros: .. code-block:: c - #define PyMyType_CheckExact(op) (Py_TYPE(op) == &PyMyType_Type) - #define PyMyType_Check(op) PyObject_TypeCheck(op, &PyMyType_Type) + #define PyMyType_CheckExact(op) (Py_TYPE(op) == &PyMyType_Type) /* Exact match. */ + #define PyMyType_Check(op) PyObject_TypeCheck(op, &PyMyType_Type) /* Exact or derived. */ -Incidentially ``PyObject_TypeCheck`` is defined as: +Incidentally ``PyObject_TypeCheck`` is defined as: .. code-block:: c #define PyObject_TypeCheck(ob, tp) \ (Py_TYPE(ob) == (tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) +.. index:: + single: Exceptions; Specialised Exceptions + --------------------------------- -Creating Specialised Excpetions +Creating Specialised Exceptions --------------------------------- -Often you need to create an Exception class that is specialised to a particular module. This can be done quite easily using either the ``PyErr_NewException`` or the ``PyErr_NewExceptionWithDoc`` functions. These create new exception classes that can be added to a module. For example: +Often you need to create an Exception class that is specialised to a particular module. +The following C code is equivalent to the Python code: + +.. code-block:: python + + class ExceptionBase(Exception): + pass + + class SpecialisedError(ExceptionBase): + pass + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Declaring Specialised Exceptions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Firstly declare the ``PyObject *`` exception: .. code-block:: c - /* Exception types as static to be initialised during module initialisation. */ - static PyObject *ExceptionBase; - static PyObject *SpecialisedError; - - /* Standard module initialisation: */ - static PyModuleDef noddymodule = { - PyModuleDef_HEAD_INIT, - "noddy", - "Example module that creates an extension type.", - -1, - NULL, NULL, NULL, NULL, NULL + /** Specialise exceptions base exception. */ + static PyObject *ExceptionBase = NULL; + /** Specialise exceptions derived from base exception. */ + static PyObject *SpecialisedError = NULL; + + /* NOTE: Functions that might raise one of these exceptions will go here. See below. */ + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Example Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now define the module, ``cExceptions_methods`` is explained later: + +.. code-block:: c + + static PyModuleDef cExceptions_module = { + PyModuleDef_HEAD_INIT, + "cExceptions", + "Examples of raising exceptions.", + -1, + cExceptions_methods, + NULL, NULL, NULL, NULL, }; - PyMODINIT_FUNC - PyInit_noddy(void) - { - PyObject* m; +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Initialising Specialised Exceptions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - noddy_NoddyType.tp_new = PyType_GenericNew; - if (PyType_Ready(&noddy_NoddyType) < 0) - return NULL; +This can be done quite easily using either the ``PyErr_NewException`` or the ``PyErr_NewExceptionWithDoc`` functions. +These create new exception classes that can be added to a module. +For example, initialise the module, this registers the exception types and the class hierarchy: - m = PyModule_Create(&noddymodule); - if (m == NULL) - return NULL; +.. code-block:: c - Py_INCREF(&noddy_NoddyType); - PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType); - + PyMODINIT_FUNC + PyInit_cExceptions(void) { + PyObject *m = PyModule_Create(&cExceptions_module); + if (m == NULL) { + return NULL; + } /* Initialise exceptions here. * * Firstly a base class exception that inherits from the builtin Exception. - * This is acheieved by passing NULL as the PyObject* as the third argument. + * This is achieved by passing NULL as the PyObject* as the third argument. * * PyErr_NewExceptionWithDoc returns a new reference. */ ExceptionBase = PyErr_NewExceptionWithDoc( - "noddy.ExceptionBase", /* char *name */ - "Base exception class for the noddy module.", /* char *doc */ - NULL, /* PyObject *base */ - NULL /* PyObject *dict */); + "cExceptions.ExceptionBase", /* char *name */ + "Base exception class for the module.", /* char *doc */ + NULL, /* PyObject *base, resolves to PyExc_Exception. */ + NULL /* PyObject *dict */); /* Error checking: this is oversimplified as it should decref * anything created above such as m. */ - if (! ExceptionBase) { + if (!ExceptionBase) { return NULL; } else { + Py_INCREF(ExceptionBase); PyModule_AddObject(m, "ExceptionBase", ExceptionBase); } - /* Now a sub-class exception that inherits from the base exception above. - * This is acheieved by passing non-NULL as the PyObject* as the third argument. + /* Now a subclass exception that inherits from the base exception above. + * This is achieved by passing non-NULL as the PyObject* as the third argument. * * PyErr_NewExceptionWithDoc returns a new reference. */ SpecialisedError = PyErr_NewExceptionWithDoc( - "noddy.SpecialsiedError", /* char *name */ - "Some specialised problem description here.", /* char *doc */ - ExceptionBase, /* PyObject *base */ - NULL /* PyObject *dict */); - if (! SpecialisedError) { + "cExceptions.SpecialsiedError", /* char *name */ + "Some specialised problem description here.", /* char *doc */ + ExceptionBase, /* PyObject *base, declared above. */ + NULL /* PyObject *dict */); + if (!SpecialisedError) { return NULL; } else { + Py_INCREF(SpecialisedError); PyModule_AddObject(m, "SpecialisedError", SpecialisedError); } /* END: Initialise exceptions here. */ - return m; } +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Raising Specialise Exceptions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + To illustrate how you raise one of these exceptions suppose we have a function to test raising one of these exceptions: .. code-block:: c static PyMethodDef Noddy_module_methods[] = { - ... - {"_test_raise", (PyCFunction)Noddy__test_raise, METH_NOARGS, "Raises a SpecialisedError."}, - ... + // ... + { + "raise_exception_base", + (PyCFunction) raise_exception_base, + METH_NOARGS, + "Raises a ExceptionBase." + }, + { + "raise_specialised_error", + (PyCFunction) raise_specialised_error, + METH_NOARGS, + "Raises a SpecialisedError." + }, + // ... {NULL, NULL, 0, NULL} /* Sentinel */ }; - We can either access the exception type directly: .. code-block:: c - static PyObject *Noddy__test_raise(PyObject *_mod/* Unused */) - { + /** Raises a ExceptionBase. */ + static PyObject *raise_exception_base(PyObject *Py_UNUSED(module)) { + if (ExceptionBase) { + PyErr_Format(ExceptionBase, "One %d two %d three %d.", 1, 2, 3); + } else { + PyErr_SetString( + PyExc_RuntimeError, + "Can not raise exception, module not initialised correctly" + ); + } + return NULL; + } + + /** Raises a SpecialisedError. */ + static PyObject *raise_specialised_error(PyObject *Py_UNUSED(module)) { if (SpecialisedError) { PyErr_Format(SpecialisedError, "One %d two %d three %d.", 1, 2, 3); } else { - PyErr_SetString(PyExc_RuntimeError, "Can not raise exception, module not initialised correctly"); + PyErr_SetString( + PyExc_RuntimeError, + "Can not raise exception, module not initialised correctly" + ); } return NULL; } - Or fish it out of the module (this will be slower): .. code-block:: c - static PyObject *Noddy__test_raise(PyObject *mod) + static PyObject *raise_specialised_error(PyObject *module) { - PyObject *err = PyDict_GetItemString(PyModule_GetDict(mod), "SpecialisedError"); + PyObject *err = PyDict_GetItemString(PyModule_GetDict(module), "SpecialisedError"); if (err) { PyErr_Format(err, "One %d two %d three %d.", 1, 2, 3); } else { @@ -247,3 +332,20 @@ Or fish it out of the module (this will be slower): } return NULL; } + +Here is some test code from ``tests/unit/test_c_exceptions.py``: + +.. code-block:: python + + from cPyExtPatt import cExceptions + + def test_raise_exception_base(): + with pytest.raises(cExceptions.ExceptionBase) as err: + cExceptions.raise_exception_base() + assert err.value.args[0] == 'One 1 two 2 three 3.' + + + def test_raise_specialised_error(): + with pytest.raises(cExceptions.SpecialisedError) as err: + cExceptions.raise_specialised_error() + assert err.value.args[0] == 'One 1 two 2 three 3.' diff --git a/doc/sphinx/source/files.rst b/doc/sphinx/source/files.rst new file mode 100644 index 0000000..30284d8 --- /dev/null +++ b/doc/sphinx/source/files.rst @@ -0,0 +1,405 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 2 + +.. index:: + single: File Paths and Files + +.. + Links, mostly to the Python documentation. + +.. _PyUnicode_FSConverter(): https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_FSConverter +.. _PyUnicode_FSDecoder(): https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_FSDecoder +.. _PyUnicode_DecodeFSDefaultAndSize(): https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_DecodeFSDefaultAndSize +.. _PyUnicode_DecodeFSDefault(): https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_DecodeFSDefault +.. _PyUnicode_EncodeFSDefault(): https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_EncodeFSDefault + +.. _PyArg_ParseTupleAndKeywords(): https://docs.python.org/3/c-api/arg.html#c.PyArg_ParseTupleAndKeywords + +********************** +File Paths and Files +********************** + +This chapter describes reading and writing files from CPython C extensions. + +.. index:: + single: File Paths + single: File Path Codecs; PyUnicode_FSConverter() + single: File Path Codecs; PyUnicode_FSDecoder() + single: File Path Codecs; PyUnicode_DecodeFSDefaultAndSize() + single: File Path Codecs; PyUnicode_DecodeFSDefault() + single: File Path Codecs; PyUnicode_EncodeFSDefault() + single: PyUnicode_FSConverter() + single: PyUnicode_FSDecoder() + single: PyUnicode_DecodeFSDefaultAndSize() + single: PyUnicode_DecodeFSDefault() + single: PyUnicode_EncodeFSDefault() + + +==================================== +File Paths +==================================== + +There are several builtin functions that allow conversion between Python and C described in the +`File System Encoding `_ +API which uses the +`filesystem encoding and error handler `_, +see also `File Objects `_ + +In summary: + +- `PyUnicode_FSConverter()`_ Converts a Python a ``str`` or *path-like* object to a Python ``bytes`` object. +- `PyUnicode_FSDecoder()`_ Converts a Python ``bytes`` object to a Python ``str``. +- `PyUnicode_DecodeFSDefaultAndSize()`_ Takes a C string and length and returns a Python ``str``. +- `PyUnicode_DecodeFSDefault()`_ Takes a null terminated C string and length and returns a Python ``str``. +- `PyUnicode_EncodeFSDefault()`_ Takes a Python ``str`` and return a Python ``bytes`` object. + +The example code is in: + +- ``src/cpy/cFile.cpp`` +- ``src/cpy/PythonFileWrapper.h`` +- ``src/cpy/PythonFileWrapper.cpp`` + +The Python tests are in ``tests/unit/test_c_file.py``. + +.. index:: + single: File Paths; Parsing Arguments + +---------------------------------------- +Parsing File Paths as Arguments +---------------------------------------- + +The Python API provides functionality for converting Python file paths (a ``str`` or *path-like* object) +to C file paths (``char *``). +From Python to C; `PyUnicode_FSConverter()`_ +and the reverse from C to Python `PyUnicode_DecodeFSDefaultAndSize()`_ + +Here is an example of taking a Python Unicode string representing a file path, converting it to C and then back +to Python. The stages are: + +- Use `PyArg_ParseTupleAndKeywords()`_ and `PyUnicode_FSConverter()`_ to convert the path-like Python object to + a Python ``bytes`` object. Note the use of the ``"O&"`` formatting string that takes a Python object and a + conversion function. +- Extract the raws bytes to use as a C path. +- Take a C path and convert it to a Python Unicode ``str`` and return it. + +The Python signature is:: + + def parse_filesystem_argument(path: typing.Union[str, pathlib.Path]) -> str: + +Here is the C code: + +.. code-block:: c + + static PyObject * + parse_filesystem_argument(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + assert(!PyErr_Occurred()); + assert(args || kwargs); + + PyBytesObject *py_path = NULL; + char *c_path = NULL; + Py_ssize_t path_size; + PyObject *ret = NULL; + + /* Parse arguments */ + static char *kwlist[] = {"path", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&", kwlist, PyUnicode_FSConverter, + &py_path)) { + goto except; + } + /* Check arguments. */ + assert(py_path); + /* Grab a reference to the internal bytes buffer. */ + if (PyBytes_AsStringAndSize((PyObject *) py_path, &c_path, &path_size)) { + /* Should have a TypeError or ValueError. */ + assert(PyErr_Occurred()); + assert(PyErr_ExceptionMatches(PyExc_TypeError) + || PyErr_ExceptionMatches(PyExc_ValueError)); + goto except; + } + assert(c_path); + /* Use the C path. */ + + /* Now convert the C path to a Python object, a string. */ + ret = PyUnicode_DecodeFSDefaultAndSize(c_path, path_size); + if (!ret) { + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + // Decref temporary locals. + Py_XDECREF(py_path); + return ret; + } + +.. index:: + single: Files + +============================= +Files +============================= + +This section describes how to interoperate between Python files, C ``FILE*`` and C++ ``iostream`` objects. + + +.. index:: + single: Files; Python Files; Reading + +---------------------------- +Reading a Python File +---------------------------- + +Here is an example of reading from a Python file in C. +The Python signature is:: + + def read_python_file_to_c(file_object: typing.IO, size: int = -1) -> bytes: + +The technique is to get the ``read()`` method from the file object with ``PyObject_GetAttrString`` then call it with the +appropriate arguments using ``PyObject_Call``. +Here is the C code: + +.. code-block:: c + + static PyObject * + read_python_file_to_c(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static const char *kwlist[] = {"file_object", "size", NULL}; + PyObject *py_file_object = NULL; + Py_ssize_t bytes_to_read = -1; + PyObject *py_read_meth = NULL; + PyObject *py_read_args = NULL; + PyObject *py_read_data = NULL; + char *c_bytes_data = NULL; + PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|n", (char **) (kwlist), + &py_file_object, &bytes_to_read)) { + return NULL; + } + // Check that this is a readable file, well does it have a read method? + /* Get the read method of the passed object */ + py_read_meth = PyObject_GetAttrString(py_file_object, "read"); // New reference + if (py_read_meth == NULL) { + PyErr_Format(PyExc_ValueError, + "Argument of type %s does not have a read() method.", + Py_TYPE(py_file_object)->tp_name); + goto except; + } + if (!PyCallable_Check(py_read_meth)) { + PyErr_Format(PyExc_ValueError, + "read attribute of type %s is not callable.", + Py_TYPE(py_file_object)->tp_name); + goto except; + } + // Call read(VisibleRecord::NUMBER_OF_HEADER_BYTES) to get a Python bytes object. + py_read_args = Py_BuildValue("(i)", bytes_to_read); + if (!py_read_args) { + goto except; + } + // This should advance that readable file pointer. + py_read_data = PyObject_Call(py_read_meth, py_read_args, NULL); + if (py_read_data == NULL) { + goto except; + } + /* Check for EOF */ + if (bytes_to_read >= 0 && PySequence_Length(py_read_data) != bytes_to_read) { + assert(PyErr_Occurred()); + PyErr_Format(PyExc_IOError, + "Reading file object gives EOF. Requested bytes %ld, got %ld.", + bytes_to_read, PySequence_Length(py_read_data)); + goto except; + } + c_bytes_data = PyBytes_AsString(py_read_data); + if (c_bytes_data == NULL) { + // TypeError already set. + goto except; + } + ret = py_read_data; + goto finally; + except: + /* Handle every abnormal condition and clean up. */ + assert(PyErr_Occurred()); + ret = NULL; + finally: + /* Clean up under normal conditions and return an appropriate value. */ + Py_XDECREF(py_read_meth); + Py_XDECREF(py_read_args); + return ret; + } + +.. index:: + single: Files; Python Files; Writing + +---------------------------- +Writing to a Python File +---------------------------- + +A similar technique can be used to write to a file, however there are a couple of C functions for writing directly to a +Python file: + +``PyFile_WriteObject()`` +---------------------------------- + +This writes a Python object to a Python file using the objects ``__str__`` method +(if `Py_PRINT_RAW `_ is given as the flags argument or +the objects ``__repr__`` method if flags is zero. + + +``PyFile_WriteString()`` +---------------------------------- + +This will write a C ``char *`` to a Python file. + +.. note:: + + ``PyFile_WriteString()`` creates a unicode string and then calls ``PyFile_WriteObject()`` + so the Python file object must be capable of writing strings. + +Here is an example of taking a Python bytes object, extracting the ``char *`` C buffer and writing that to a Python +file. +The Python function signature is:: + + def write_bytes_to_python_file(bytes_to_write: bytes, file_object: typing.IO) -> int: + +Here is the C code: + +.. code-block:: c + + static PyObject * + write_bytes_to_python_file(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static const char *kwlist[] = {"bytes_to_write", "file_object", NULL}; + PyObject *py_file_object = NULL; + Py_buffer c_buffer; + PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "y*O", (char **) (kwlist), + &c_buffer, &py_file_object)) { + return NULL; + } + /* NOTE: PyFile_WriteString() creates a unicode string and then + * calls PyFile_WriteObject() so the py_file_object must be + * capable of writing strings. */ + int result = PyFile_WriteString((char *)c_buffer.buf, py_file_object); + if (result != 0) { + goto except; + } + ret = Py_BuildValue("n", c_buffer.len); + goto finally; + except: + assert(PyErr_Occurred()); + ret = NULL; + finally: + return ret; + } + + +.. index:: + single: Files; Python Files; C++ Wrapper + +A C++ Python File Wrapper +---------------------------------- + +In ``src/cpy/PythonFileWrapper.h`` and ``src/cpy/PythonFileWrapper.cpp`` there is a C++ class that takes a Python file +and extracts the ``read()``, ``write()``, ``seek()`` and ``tell()`` methods that can then be used to read and write to +the Python file from C++. + +Here is the class: + +.. code-block:: c++ + + /// Class that is created with a PyObject* that looks like a Python File. + /// This can then read from that file object ans write to a user provided C++ stream or read from a user provided C++ + /// stream and write to the give Python file like object. + class PythonFileObjectWrapper { + public: + explicit PythonFileObjectWrapper(PyObject *python_file_object); + + /// Read from a Python file and write to the C++ stream. + /// Return zero on success, non-zero on failure. + int read_py_write_cpp(Py_ssize_t number_of_bytes, std::iostream &ios); + + /// Read from a C++ stream and write to a Python file. + /// Return zero on success, non-zero on failure. + int read_cpp_write_py(std::iostream &ios, Py_ssize_t number_of_bytes); + + /// Read a number of bytes from a Python file and load them into the result. + /// Return zero on success, non-zero on failure. + int read(Py_ssize_t number_of_bytes, std::vector &result); + + /// Write a number of bytes to a Python file. + /// Return zero on success, non-zero on failure. + int write(const char *buffer, Py_ssize_t number_of_bytes); + + /// Move the file pointer to the given position. + /// whence is: + /// 0 – start of the stream (the default); offset should be zero or positive. + /// 1 – current stream position; offset may be negative. + /// 2 – end of the stream; offset is usually negative. + /// Returns the new absolute position. + long seek(Py_ssize_t pos, int whence = 0); + + /// Returns the current absolute position. + long tell(); + /// Returns a multi-line string that describes the class state. + std::string str_pointers(); + /// Returns a Python multi-line bytes object that describes the class state. + PyObject *py_str_pointers(); + /// Destructor, this decrements the held references. + virtual ~PythonFileObjectWrapper(); + + protected: + PyObject *m_python_file_object = NULL; + PyObject *m_python_read_method = NULL; + PyObject *m_python_write_method = NULL; + PyObject *m_python_seek_method = NULL; + PyObject *m_python_tell_method = NULL; + }; + +Some example code is in ``src/cpy/cFile.cpp``: + +.. code-block:: cpp + + /** + * Wraps a Python file object. + */ + static PyObject * + wrap_python_file(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static const char *kwlist[] = {"file_object", NULL}; + PyObject *py_file_object = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", (char **) (kwlist), + &py_file_object)) { + return NULL; + } + PythonFileObjectWrapper py_file_wrapper(py_file_object); + + /* Exercise ths wrapper by writing, reading etc. */ + py_file_wrapper.write("Test write to python file", 25); + return py_file_wrapper.py_str_pointers(); + } + +And some tests are in ``tests/unit/test_c_file.py``: + +.. code-block:: python + + def test_wrap_python_file(): + file = io.BytesIO() + result = cFile.wrap_python_file(file) + print() + print(' Result '.center(75, '-')) + print(result.decode('ascii')) + print(' Result DONE '.center(75, '-')) + print(' file.getvalue() '.center(75, '-')) + get_value = file.getvalue() + print(get_value) + print(' file.getvalue() DONE '.center(75, '-')) + assert get_value == b'Test write to python file' diff --git a/doc/sphinx/source/further_reading.rst b/doc/sphinx/source/further_reading.rst index 61c4064..9aea7c3 100644 --- a/doc/sphinx/source/further_reading.rst +++ b/doc/sphinx/source/further_reading.rst @@ -6,37 +6,45 @@ .. _further_reading: -============================================ -Further Reading -============================================ +.. index:: + single: Further Reading +*************************** +Further Reading +*************************** --------------------------------------------- -Useful Links --------------------------------------------- +Some other sources of information for you. -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -C Extensions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. index:: + single: Further Reading; General -* python.org Tutorial: `https://docs.python.org/3/extending/index.html `_ -* python.org C/C++ reference: `https://docs.python.org/3/c-api/index.html `_ -* Joe Jevnik's "How to Write and Debug C Extension Modules": - * Documentation: `https://llllllllll.github.io/c-extension-tutorial/index.html `_ - * Code: `https://github.com/llllllllll/c-extension-tutorial `_ +============================================ +C Extensions - General +============================================ +* python.org `Extension tutorial `_ +* python.org `C/C++ reference `_ +* Joe Jevnik's "How to Write and Debug C Extension Modules" + `Documentation `_ + and `Code `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Porting to Python 3 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. index:: + single: Further Reading; Books -* General, and comprehensive: `http://python3porting.com/ `_ -* Porting Python 2 code to Python 3: `https://docs.python.org/3/howto/pyporting.html `_ -* Porting C Extensions to Python 3: `https://docs.python.org/3/howto/cporting.html `_ -* ``py3c``: Python 2/3 compatibility layer for C extensions: - * Documentation: `https://py3c.readthedocs.io/en/latest/ `_ - * Project: `https://github.com/encukou/py3c `_ +============================================ +Books +============================================ +* The `CPython Internals book (RealPython) `_ +* The `Python Cookbook `_ + by David Beazley and Brian Jones. +.. index:: + single: Further Reading; Projects +============================================ +Projects +============================================ +* Python memory tracing: https://github.com/paulross/pymemtrace +* Python/C++ homogeneous containers: https://github.com/paulross/PyCppContainers diff --git a/doc/sphinx/source/index.rst b/doc/sphinx/source/index.rst index 855eb73..61ad337 100644 --- a/doc/sphinx/source/index.rst +++ b/doc/sphinx/source/index.rst @@ -6,30 +6,49 @@ Coding Patterns for Python Extensions ===================================================== -This describes reliable patterns of coding Python Extensions in C. It covers the essentials of reference counts, exceptions and creating functions that are safe and efficient. +.. Omitting test chapters that illustrate Sphinx markup: + + _headings + _index_styles + .. toctree:: - :numbered: - :maxdepth: 3 - - refcount - exceptions - canonical_function - parsing_arguments - new_types - module_globals - super_call - compiler_flags - debugging/debug - thread_safety - code_layout - cpp - miscellaneous - further_reading - - -Indices and tables + :numbered: + :maxdepth: 3 + + introduction + simple_example + refcount + containers_and_refcounts + struct_sequence + exceptions + canonical_function + parsing_arguments + new_types + module_globals + logging + files + subclassing_and_super_call + capsules + iterators_generators + context_manager + pickle + watchers + compiler_flags + debugging/debug + memory_leaks + thread_safety + code_layout + cpp + miscellaneous + install + further_reading + todo + HISTORY + +Search ================== +* :ref:`genindex` +* :ref:`modindex` * :ref:`search` - diff --git a/doc/sphinx/source/install.rst b/doc/sphinx/source/install.rst new file mode 100644 index 0000000..c96a418 --- /dev/null +++ b/doc/sphinx/source/install.rst @@ -0,0 +1,222 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + +.. highlight:: python + :linenothreshold: 30 + +.. toctree:: + :maxdepth: 3 + +.. + Links, mostly to the Python documentation. + Specific container links are just before the appropriate section. + +.. index:: + single: Installation + +.. _chapter_installation: + +====================================== +Installation +====================================== + +This project is primarily +`a documentation project `_ +however it does contain a lot of code examples and tests. + +Project Links +============= + +- Source is `on GitHub `_. +- Documentation `Read the Docs `_. +- Project is `on PyPi `_. + +This code can be installed as follows. + +Setup +===== +First make a virtual environment in your :file:`{}`, say :file:`{~/pyvenvs}`: + +.. code-block:: console + + $ python3 -m venv /cPyExtPatt + $ source /cPyExtPatt/bin/activate + (cPyExtPatt) $ + +Stable release +============== + +To install ``cPyExtPatt``, run this command in your terminal: + +.. code-block:: console + + $ pip install cPyExtPatt + +This is the preferred method to install this project, as it will always install the most recent stable release. + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + + +From sources +============ + +The sources for cpip can be downloaded from the `Github repo`_. + +You can clone the public repository: + +.. code-block:: console + + (cPyExtPatt) $ git clone https://github.com/paulross/PythonExtensionPatterns.git + +Install Requirements +==================== + +Then install the requirements in your virtual environment (these are pretty minimal): + +.. code-block:: console + + (cPyExtPatt) $ pip install -r requirement.txt + +Once you have a copy of the source, you can install it with: + +.. code-block:: console + + (cPyExtPatt) $ python setup.py install + +Or: + +.. code-block:: console + + (cPyExtPatt) $ python setup.py develop + +As you prefer. + +What is in the Package +====================== + +Within the ``cPyExtPatt`` package are several modules. + +Any module can be imported with: + +.. code-block:: python + + from cPyExtPatt import + +The modules are: + +=========================== ================================================================= +Module Description +=========================== ================================================================= +``cExceptions`` Examples of creating and using exceptions. +``cModuleGlobals`` Accessing module globals. +``cObject`` Exploring ``PyObject``. +``cParseArgs`` Parsing Python function arguments in all their ways. +``cParseArgsHelper`` More parsing Python function arguments. +``cPyRefs`` Exploring reference counts. +``cPickle`` Pickling objects. +``cFile`` Working with Python and C files. +``Capsules.spam`` A simple capsule. +``Capsules.spam_capsule`` A target capsule. +``Capsules.spam_client`` A client capsule. +``Capsules.datetimetz`` Using an existing capsule from Python's stdlib. +``cpp.placement_new`` Example of using C++ placement new. +``cpp.cUnicode`` Example of working with unicode to and from C++ and Python. +``SimpleExample.cFibA`` A simple example of a Python C extension. +``SimpleExample.cFibB`` A simple example of a Python C extension. +``Iterators.cIterator`` Iterators in C. +``SubClass.sublist`` Subclassing, in this case a list. +``Threads.csublist`` Illustrates thread contention in C. +``Threads.cppsublist`` Illustrates thread contention in C++. +``Logging.cLogging`` Examples of logging. +``cRefCount`` Reference count explorations. +``cCtxMgr`` Example of a context manager. +``cStructSequence`` Example of a named tuple in C. +``cWatchers`` Example of a dictionary watcher in C. +=========================== ================================================================= + +In addition there are these modules availlable in Python 3.12+: + +=========================== ================================================================= +Module Description +=========================== ================================================================= +``cWatchers`` Examples of watchers. +=========================== ================================================================= + +Running the Tests +==================== + +Then you should be able to run the tests: + +.. code-block:: console + + (cPyExtPatt) $ pytest tests/ + ================================ test session starts ================================ + platform darwin -- Python 3.13.1, pytest-8.3.4, pluggy-1.5.0 + rootdir: /Users/engun/GitHub/paulross/PythonExtensionPatterns + collected 275 items + + tests/unit/test_c_capsules.py ..s..................s. [ 8%] + tests/unit/test_c_cpp.py ............. [ 13%] + tests/unit/test_c_ctxmgr.py ..... [ 14%] + tests/unit/test_c_custom_pickle.py ..... [ 16%] + tests/unit/test_c_exceptions.py .s...s...... [ 21%] + tests/unit/test_c_file.py .....ss.......s [ 26%] + tests/unit/test_c_iterators.py .s.s............... [ 33%] + tests/unit/test_c_logging.py .. [ 34%] + tests/unit/test_c_module_globals.py ....... [ 36%] + tests/unit/test_c_object.py .s..s..sss. [ 40%] + tests/unit/test_c_parse_args.py ...s..s.......ss.....................sssssss. [ 57%] + ..........ssss..... [ 64%] + tests/unit/test_c_parse_args_helper.py ..... [ 65%] + tests/unit/test_c_py_refs.py ..... [ 67%] + tests/unit/test_c_ref_count.py s............................................. [ 84%] + .......... [ 88%] + tests/unit/test_c_simple_example.py .......... [ 91%] + tests/unit/test_c_struct_sequence.py . [ 92%] + tests/unit/test_c_subclass.py .s... [ 93%] + tests/unit/test_c_threads.py ..s.s............ [100%] + + ========================= 242 passed, 33 skipped in 33.99s ========================== + +The skipped tests are specific to a Python version that is not the current version in your virtual environment. + +Building the Documentation +========================== + +If you want to build the documentation you need to: + +.. code-block:: console + + (cPyExtPatt) $ cd doc/sphinx + (cPyExtPatt) $ make html latexpdf + +This takes about 40 seconds from clean. + +The landing page is *build/html/index.html* in *doc/sphinx*. +The PDF is *build/latex/PythonExtensionPatterns.pdf* in *doc/sphinx*. + +Building Everything +========================== + +At the project root there is a script ``build_all.sh`` which, for every supported version of Python: + +- Builds and tests the C/C++ code. +- Creates a Python virtual environment (optionally deleting any existing one). +- Run ``pip install -r requirements.txt`` on the virtual environment. +- Run ``python setup.py develop`` in that virtual environment. +- Run ``pytest tests/``. +- Run ``python setup.py bdist_wheel``. +- Run ``python setup.py sdist``. +- Optionally, create the documentation. +- Report the results. + +The script will halt on the first error returning the error code. + +Takes about 70 seconds per Python version. + +.. _Github repo: https://github.com/paulross/PythonExtensionPatterns +.. _zip: https://github.com/paulross/PythonExtensionPatterns/archive/refs/heads/master.zip diff --git a/doc/sphinx/source/introduction.rst b/doc/sphinx/source/introduction.rst new file mode 100644 index 0000000..fa0f746 --- /dev/null +++ b/doc/sphinx/source/introduction.rst @@ -0,0 +1,224 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + +============ +Introduction +============ + +This projects explores reliable patterns of coding Python Extensions in C. +It covers the essentials of reference counts, exceptions and creating objects and functions that are safe and +efficient. + +Writing Python C Extensions can be daunting; you have to cast aside the security and fluidity of Python and embrace C, +not just C but Pythons C API, which is huge [#]_ and changes between versions [#]_. +Not only do you have to worry about just your standard ``malloc()`` and ``free()`` cases but now you have to contend +with how CPython's does its memory management which is by *reference counting*. + +I describe some of the pitfalls you (I am thinking of you as a savvy C coder) can encounter and some of the coding +patterns that you can use to avoid them. + +This also might help if you are considering code to submit to the Python standard library which depends extensively on +C extensions. + +This is primarily a documentation project, with code. +If you want that code see :ref:`chapter_installation`, otherwise read on. + +.. index:: + single: Personal Note + +--------------------- +A Personal Note +--------------------- + +This project has its roots when, long ago, I joined a tech company that had created many production +critical Python extensions in C. +These were all written by a single engineer who left shortly after I joined. +The CTO appointed me as the replacement on the dubious basis that I knew Python and C although I had never written a +Python extension in C. + +I really struggled to bring my knowledge of both languages to their very complicated, and crucial, codebase. +To be honest I don't think I did a great job, but as I was the 'owner' I somehow got away with it. + +After some time it occurred to me that, rather learning from their scrappy CPython C code, +I asked myself "how would you write a Python C Extension from scratch?". +So on the commute and at weekends I did just that and slowly things became clearer. +This eventually lead me to being invited to PyConUS to give a talk about the subject. +This document is a synthesis of the latter journey which ended up giving me far more confidence about the subject than +during my earlier difficulties. + +My fond hope is that you will find that this document makes it much easier to work in this field than I found initially. +Another way of saying that is that I dedicate this document to you, and your work. + +So why write Python C Extensions? + +--------------------- +Firstly Why? +--------------------- + +There are several reasons why you might want to write a C extension: + +^^^^^^^^^^^^^^^^^^^ +Performance +^^^^^^^^^^^^^^^^^^^ + +This is the most compelling reason, a 50x or 100x improvement over pure Python is not unusual. + +.. index:: + single: C/C++; Libraries + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Interface with C/C++ libraries +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you have a library in C or C++ you will have to write a C extension to give a Python interface to that library. + +.. index:: + single: Memory Usage + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Less memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Python is pretty memory hungry, for example a float in Python is 24 bytes, the corresponding double in C is 8 bytes. +C and C++ have more specific deallocation policies than with a garbage collected language. + +.. index:: + single: GIL + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +C Extensions do not need the Global Interpreter Lock (GIL) when working with C/C++ code which do *not* make calls to +the CPython API. + +--------------------- +Why Not? +--------------------- + +Like everything, there are disadvantages, here are some: + +- C Extensions are something of a niche skill. + Achieving and maintaining that skill has costs. + Hopefully this project reduces those. +- Writing C Extensions is more time consuming than pure Python. + There is also the intellectual problem that you are dealing with Pure C/CPython/Python code which expects + a lot of context switching. + This project proposes patterns of code that should reduce the cognitive overhead of all of that. +- Testing C Extensions, whilst excellent at a high level, can be really tricky at a line-of-code level. + +------------------------------------ +Alternatives to C Extensions +------------------------------------ + +There are several alternatives to writing an extension directly in C, here are some: + +.. index:: + single: ctypes + +^^^^^^^^^^^^^^^^^^^ +``ctypes`` +^^^^^^^^^^^^^^^^^^^ + +`ctypes `_ is a well documented foreign function library +for Python and is part of Python's standard library. +The module allows direct access to C/C++ libraries (such as ``libc``). +If you need this functionality, for example you need to access a binary library where you do not have the original +source code so you can not build the library into your own code. + +.. index:: + single: Code Generators + +^^^^^^^^^^^^^^^^^^^ +Code Generators +^^^^^^^^^^^^^^^^^^^ + +There are a number of projects out there that take high level code and generate C/C++ code that can then be built into +a Python module. +Some examples are: + +- `SWIG `_ is a very well established project. + An advantage that distinguishes it from other projects is its multi-language support. +- `Cython `_ is another well established project. + You write in Python-like pseudo code that is translated into C which is then compiled into a Python module. + A notable feature is its excellent support for working with ``numpy``. + If you are using Cython you might find another project of mine, + `Notes on Cython `_, useful. +- `PyBind11 `_ is an excellent and ingenious project that uses C++ template to do + the bulk of the work in generating code for a Python module. + +There are common drawbacks of code generators: + +- The testing and debug story is generally poor. +- There is occasional pathological behaviour where a small change or version upgrade can introduce a large performance + degradation. +- If you are crossing the boundary between the Python interpreter and compiled C/C++ at a high frequency, perhaps with + many small objects, code generators can create a performance overhead compared to C extensions. + An example is shown here with my project on `XML creation `_. + + +There are many other alternatives such as ``pypy``, ``numba`` that are worth knowing about. + +------------------------------------ +A Faustian Bargain +------------------------------------ + +The ability to write C code and link it to the Python runtime has played a huge part in Python's success story. +Much of the stdlib, and third party packages like ``numpy`` gives C like performance with Python's simple interface. +This gives the reference implementation, CPython, huge power and ease of use. + +However the downside is that it becomes very difficult to create *alternative* implementations of the Python language +and, who knows, these might be faster, use less memory or have better parallelism. +This is because any alternate implementation must work the thousands of CPython C extensions out there, with all their +quirks, to have any chance of being compatible with existing code. + +Python C extensions will be around for a long time. +It is a skill worth learning. + +.. _introduction_summary_advice: + +------------------------------------ +Summary Advice +------------------------------------ + +My advice if you are thinking about extensions: + +- They can be really powerful, 100x powerful +- They can be expensive to write and maintain +- It helps to follow established patterns +- Write everything in Python, benchmark/profile before deciding what to put into CPython +- Use them for low level, stable, library code +- Keep the CPython layer as thin as possible +- Testing, testing testing! + +.. index:: + single: Documentation Lacunae; General + +--------------------------------- +Python Documentation +--------------------------------- + +Some of the official Python documentation is wrong, misleading or missing and this project goes some way to correcting +that. +For example see the chapters :ref:`chapter_containers_and_refcounts`, :ref:`chapter_struct_sequence` +and :ref:`chapter_creating_new_types`. +There is an index entry "Documentation Lacunae" that identifies sections that improve or correct the official +Python documentation. + +--------------------------------- +Project Links +--------------------------------- + +- Source is `on GitHub `_. +- Documentation `Read the Docs `_. +- Project is `on PyPi `_. + +Next up: a simple example showing the effect on code performance. + +.. rubric:: Footnotes + +.. [#] Huge, but pretty consistent once mastered. +.. [#] Version 0.3 of this project supports Python versions: 3.9, 3.10, 3.11, 3.12, 3.13. diff --git a/doc/sphinx/source/iterators_generators.rst b/doc/sphinx/source/iterators_generators.rst new file mode 100644 index 0000000..fa23e21 --- /dev/null +++ b/doc/sphinx/source/iterators_generators.rst @@ -0,0 +1,810 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + +.. _chapter_iterators_generators: + +.. + Links, mostly to the Python documentation. + +.. _Generator: https://docs.python.org/3/glossary.html#term-generator + + +*************************** +Iterators and Generators +*************************** + +This chapter describes how to write iterators for your C objects. +These iterators allow your objects to be used with a `Generator`_. + +.. index:: + single: Iterators + +=========================== +Iterators +=========================== + +The iterator concept is actually fairly straight forward: + +- You have some object that contains some data. +- You have some iterator that traverses the data. + +That iterator: + +- Has a strong reference to the originating object, thus its data. + This strong reference keeps the originating object alive as long as the iterator is alive. +- It has a notion of *state*, in other words 'where I was before so know where to go next'. + +.. warning:: + + The strong reference to the underlying data structure keeps it alive but what happens if the underlying structure + is altered *during* iteration? + + Here is an example: + + .. code-block:: python + + lst = list(range(8)) + for i, value in enumerate(lst): + print(f'i={i} value={value}') + del lst[i] + + This gives the sequence: + + .. code-block:: bash + + i=0 value=0 + i=1 value=2 + i=2 value=4 + i=3 value=6 + + Which may not be what you want. + It is hard to make a 'good' design to cope with this (defer the ``del``? raise?) so the general advice is: do not + alter the underlying structure whilst iterating. + +-------------------------------------- +Example of a Sequence +-------------------------------------- + +In this example we create a module that has an object which holds a sequence of C ``long`` s. +The complete code is in ``src/cpy/Iterators/cIterator.c`` here are just the essential parts. +The test code is in ``tests/unit/test_c_iterators.py``. + +Essentially in Python this means I will be able do this: + +.. code-block:: python + + from cPyExtPatt.Iterators import cIterator + + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = [v for v in sequence] + assert result == [1, 7, 4] + +.. note:: + + Because of the entwined nature of the sequence object and the iterator object the code in + ``src/cpy/Iterators/cIterator.c`` occasionally appears out of order. + +Firstly, here is the C declaration of the ``SequenceOfLong`` struct: + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + + #include + #include "structmember.h" + + typedef struct { + PyObject_HEAD + long *array_long; + size_t size; + } SequenceOfLong; + +This will be instantiated with a Python sequence of integers: + +.. code-block:: c + + static PyObject * + SequenceOfLong_new(PyTypeObject *type, PyObject *Py_UNUSED(args), + PyObject *Py_UNUSED(kwds)) { + SequenceOfLong *self; + self = (SequenceOfLong *) type->tp_alloc(type, 0); + if (self != NULL) { + assert(!PyErr_Occurred()); + self->size = 0; + self->array_long = NULL; + } + return (PyObject *) self; + } + +And initialised with a Python sequence of integers: + +.. note:: + + The use of the `Sequence Protocol `_ API such as + `PySequence_Length() `_ + and `PySequence_GetItem() `_ + +.. code-block:: c + + static int + SequenceOfLong_init(SequenceOfLong *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return -1; + } + if (!PySequence_Check(sequence)) { + return -2; + } + self->size = PySequence_Length(sequence); + self->array_long = malloc(self->size * sizeof(long)); + if (! self->array_long) { + return -3; + } + for (Py_ssize_t i = 0; i < PySequence_Length(sequence); ++i) { + // New reference. + PyObject *py_value = PySequence_GetItem(sequence, i); + if (PyLong_Check(py_value)) { + self->array_long[i] = PyLong_AsLong(py_value); + Py_DECREF(py_value); + } else { + PyErr_Format( + PyExc_TypeError, + "Argument [%zd] must be a int, not type %s", + i, + Py_TYPE(sequence)->tp_name + ); + // Clean up on error. + free(self->array_long); + self->array_long = NULL; + Py_DECREF(py_value); + return -4; + } + } + return 0; + } + +And the de-allocation function frees the dynamically allocated memory: + +.. code-block:: c + + static void + SequenceOfLong_dealloc(SequenceOfLong *self) { + free(self->array_long); + Py_TYPE(self)->tp_free((PyObject *) self); + } + +We provide a single method ``size()`` for the length of the sequence and a ``__str__`` method: + +.. code-block:: c + + SequenceOfLong_size(SequenceOfLong *self, PyObject *Py_UNUSED(ignored)) { + return Py_BuildValue("n", self->size); + } + + static PyMethodDef SequenceOfLong_methods[] = { + { + "size", + (PyCFunction) SequenceOfLong_size, + METH_NOARGS, + "Return the size of the sequence." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static PyObject * + SequenceOfLong___str__(SequenceOfLong *self, PyObject *Py_UNUSED(ignored)) { + assert(!PyErr_Occurred()); + return PyUnicode_FromFormat("", self->size); + } + +The type declaration then becomes: + +.. code-block:: c + + static PyTypeObject SequenceOfLongType= { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceOfLong", + .tp_basicsize = sizeof(SequenceOfLong), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceOfLong_dealloc, + .tp_str = (reprfunc) SequenceOfLong___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "Sequence of long integers.", + .tp_methods = SequenceOfLong_methods, + .tp_init = (initproc) SequenceOfLong_init, + .tp_new = SequenceOfLong_new, + }; + +This will be used thus: + +.. code-block:: python + + from cPyExtPatt.Iterators import cIterator + + sequence = cIterator.SequenceOfLong([1, 7, 4]) + +But we can't (yet) iterate across the sequence. +To do that we need to add an iterator. + +.. index:: + single: Iterators; Adding an Iterator + +-------------------------------------- +Adding an Iterator +-------------------------------------- + +Here is the iterator that takes a reference to the ``SequenceOfLong`` and maintains an index: + +.. code-block:: c + + typedef struct { + PyObject_HEAD + PyObject *sequence; + size_t index; + } SequenceOfLongIterator; + +Here are the ``__new__``, ``__init__`` and de-allocation methods: + +.. code-block:: c + + static PyObject * + SequenceOfLongIterator_new(PyTypeObject *type, PyObject *Py_UNUSED(args), + PyObject *Py_UNUSED(kwds)) { + SequenceOfLongIterator *self; + self = (SequenceOfLongIterator *) type->tp_alloc(type, 0); + if (self != NULL) { + assert(!PyErr_Occurred()); + } + return (PyObject *) self; + } + + // Forward reference + static int is_sequence_of_long_type(PyObject *op); + // Defined later to be: + // static int + // is_sequence_of_long_type(PyObject *op) { + // return Py_TYPE(op) == &SequenceOfLongType; + // } + +Here is the initialisation function, note the line ``Py_INCREF(sequence);`` that keeps the original sequence alive. + +.. code-block:: c + + static int + SequenceOfLongIterator_init(SequenceOfLongIterator *self, PyObject *args, + PyObject *kwds) { + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return -1; + } + if (!is_sequence_of_long_type(sequence)) { + PyErr_Format( + PyExc_ValueError, + "Argument must be a SequenceOfLongType, not type %s", + Py_TYPE(sequence)->tp_name + ); + return -2; + } + // Borrowed reference + // Keep the sequence alive as long as the iterator is alive. + // Decrement on iterator de-allocation. + Py_INCREF(sequence); + self->sequence = sequence; + self->index = 0; + return 0; + } + +Here is the de-allocation function once the iterator is deleted. +Note the line ``Py_XDECREF(self->sequence);`` that allows the original sequence to be free'd. + +.. code-block:: c + + static void + SequenceOfLongIterator_dealloc(SequenceOfLongIterator *self) { + // Decrement borrowed reference. + Py_XDECREF(self->sequence); + Py_TYPE(self)->tp_free((PyObject *) self); + } + +Here is the ``__next__`` method. +This returns the next object in the sequence, or NULL if the sequence is exhausted. +It updates the internal counter on each call: + +.. code-block:: c + + static PyObject * + SequenceOfLongIterator_next(SequenceOfLongIterator *self) { + size_t size = ((SequenceOfLong *) self->sequence)->size; + if (self->index < size) { + PyObject *ret = PyLong_FromLong( + ((SequenceOfLong *) self->sequence)->array_long[self->index] + ); + self->index += 1; + return ret; + } + // End iteration. + return NULL; + } + +Here is the iterator type declaration, note the use of +`tp_iter `_ +and `tp_iternext `_. + +- ``tp_iter`` signals that this object is an iterable and its result is the iterator. + The use of ``PyObject_SelfIter`` merely says "I am an iterator". +- ``tp_iternext`` is the function to call with the iterator as its sole argument + and this returns the next item in the sequence. + +.. note:: + + `PyObject_SelfIter() `_ + is a supported CPython API and is implemented thus: + + .. code-block:: c + + PyObject * + PyObject_SelfIter(PyObject *obj) + { + Py_INCREF(obj); + return obj; + } + +Here is the type declaration for the iterator, the iterator is both iterable and has a ``__next__`` method +so both ``tp_iter`` and ``tp_iternext`` are defined: + +.. code-block:: c + + static PyTypeObject SequenceOfLongIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceOfLongIterator", + .tp_basicsize = sizeof(SequenceOfLongIterator), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceOfLongIterator_dealloc, + .tp_str = (reprfunc) SequenceOfLongIterator___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "SequenceOfLongIterator object.", + .tp_iter = PyObject_SelfIter, + .tp_iternext = (iternextfunc) SequenceOfLongIterator_next, + .tp_init = (initproc) SequenceOfLongIterator_init, + .tp_new = SequenceOfLongIterator_new, + }; + +Now change the type declaration of the ``SequenceOfLongType`` to add iteration. +This defines ``tp_iter`` as it is iterable but does *not* define the ``tp_iternext`` method as it does *not* have +a ``__next__`` method, the iterator instance provides that: + +.. code-block:: + + static PyTypeObject SequenceOfLongType= { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceOfLong", + .tp_basicsize = sizeof(SequenceOfLong), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceOfLong_dealloc, + .tp_str = (reprfunc) SequenceOfLong___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "Sequence of long integers.", + // I am iterable, this function gives you an iterator on me. + .tp_iter = (getiterfunc) SequenceOfLong_iter, + .tp_methods = SequenceOfLong_methods, + .tp_init = (initproc) SequenceOfLong_init, + .tp_new = SequenceOfLong_new, + }; + +.. index:: + single: Iterators; Module Initialisation + +----------------------------------------- +A Note on Module Initialisation +----------------------------------------- + +The module initialisation looks like this: + +.. code-block:: c + + PyMODINIT_FUNC + PyInit_cIterator(void) { + PyObject *m; + m = PyModule_Create(&iterator_cmodule); + if (m == NULL) { + return NULL; + } + // ... + } + +Naturally enough we have to include the initialisation of ``SequenceOfLongType``: + +.. code-block:: c + + if (PyType_Ready(&SequenceOfLongType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&SequenceOfLongType); + if (PyModule_AddObject( + m, + "SequenceOfLong", + (PyObject *) &SequenceOfLongType) < 0 + ) { + Py_DECREF(&SequenceOfLongType); + Py_DECREF(m); + return NULL; + } + +We *must* include the initialisation of ``SequenceOfLongIteratorType``: + +.. code-block:: c + + if (PyType_Ready(&SequenceOfLongIteratorType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&SequenceOfLongIteratorType); + +However the following is optional, as the comment suggests: + +.. code-block:: c + + // Not strictly necessary unless you need to expose this type. + // For type checking for example. + if (PyModule_AddObject( + m, + "SequenceOfLongIterator", + (PyObject *) &SequenceOfLongIteratorType) < 0 + ) { + Py_DECREF(&SequenceOfLongType); + Py_DECREF(&SequenceOfLongIteratorType); + Py_DECREF(m); + return NULL; + } + +If you omit that the code will work just fine, the iterator is instantiated dynamically, it is just that the type is +not exposed from the module. + +.. index:: + single: Iterators; Iterating Python In C + +------------------------------ +Iterating a Python Object in C +------------------------------ + +In ``src/cpy/Iterators/cIterator.c`` there is an example of iterating a +Python object using the +`Iterator Protocol `_. +There is a function ``iterate_and_print`` that takes an object supporting +the Iterator Protocol and iterates across it printing out each item. + +The equivalent Python code is: + +.. code-block:: python + + def iterate_and_print(sequence: typing.Iterable) -> None: + print('iterate_and_print:') + for i, item in enumerate(sequence): + print(f'[{i}]: {item}') + print('iterate_and_print: DONE') + +The C code looks like this: + +.. code-block:: c + + static PyObject * + iterate_and_print(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return NULL; + } + PyObject *iterator = PyObject_GetIter(sequence); + if (iterator == NULL) { + /* propagate error */ + assert(PyErr_Occurred()); + return NULL; + } + PyObject *item = NULL; + long index = 0; + fprintf(stdout, "%s:\n", __FUNCTION__ ); + while ((item = PyIter_Next(iterator))) { + /* do something with item */ + fprintf(stdout, "[%ld]: ", index); + if (PyObject_Print(item, stdout, Py_PRINT_RAW) == -1) { + /* Handle error. */ + Py_DECREF(item); + Py_DECREF(iterator); + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, + "Can not print an item of type %s", + Py_TYPE(sequence)->tp_name); + } + return NULL; + } + fprintf(stdout, "\n"); + ++index; + /* release reference when done */ + Py_DECREF(item); + } + Py_DECREF(iterator); + if (PyErr_Occurred()) { + /* propagate error */ + return NULL; + } + fprintf(stdout, "%s: DONE\n", __FUNCTION__ ); + assert(!PyErr_Occurred()); + Py_RETURN_NONE; + } + +This function is added to the cIterator module thus: + +.. code-block:: c + + static PyMethodDef cIterator_methods[] = { + {"iterate_and_print", (PyCFunction) iterate_and_print, METH_VARARGS, + "Iterate through the argument printing the values."}, + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + + static PyModuleDef iterator_cmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cIterator", + .m_doc = ( + "Example module that creates an extension type" + "that has forward and reverse iterators." + ), + .m_size = -1, + .m_methods = cIterator_methods, + }; + +An example of using this is shown below. + +------------------------------ +Examples +------------------------------ + +Now we can import the module, and create a sequence: + +.. code-block:: python + + from cPyExtPatt.Iterators import cIterator + + sequence = cIterator.SequenceOfLong([1, 7, 4]) + +And these calls now work: + +.. code-block:: python + + result = [v for v in sequence] + assert result == [1, 7, 4] + +Delete the underlying object, the iteration still works: + +.. code-block:: python + + iterator = iter(sequence) + del sequence + result = [v for v in iterator] + assert result == [1, 7, 4] + +Using the builtin ``next()``: + +.. code-block:: python + + iterator = iter(sequence) + assert next(iterator) == 1 + assert next(iterator) == 7 + assert next(iterator) == 4 + # next() will raise a StopIteration. + +Using the builtin ``sorted()``: + +.. code-block:: python + + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = sorted(sequence) + assert result == [1, 4, 7, ] + +And to iterate across a Python object: + +.. code-block:: python + + cIterator.iterate_and_print('abc') + +Result in the stdout: + +.. code-block:: text + + iterate_and_print: + [0]: a + [1]: b + [2]: c + iterate_and_print: DONE + +.. note:: + + If you are running under pytest you can capture the output to + stdout from C using the ``capfd`` fixture: + + + .. code-block:: python + + import pytest + + @pytest.mark.parametrize( + 'arg, expected', + ( + ( + 'abc', + """iterate_and_print: + [0]: a + [1]: b + [2]: c + iterate_and_print: DONE + """ + ), + ) + ) + def test_iterate_and_print(arg, expected, capfd): + cIterator.iterate_and_print(arg) + captured = capfd.readouterr() + assert captured.out == expected + + +.. _PySequenceMethods: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods +.. _reversed(): https://docs.python.org/3/library/functions.html#reversed +.. _\__reversed__(): https://docs.python.org/3/reference/datamodel.html#object.__reversed__ +.. _\__len__(): https://docs.python.org/3/reference/datamodel.html#object.__len__ +.. _\__getitem__(): https://docs.python.org/3/reference/datamodel.html#object.__getitem__ + +.. index:: + single: Reverse Iterators + single: Iterators; Reverse + single: PySequenceMethods + single: __reversed__() + single: __len__() + single: __getitem__() + +----------------------------------------- +Reverse Iterators +----------------------------------------- + +If we try and use the `reversed()`_ function on our current ``SequenceOfLong`` we will get an error: + +.. code-block:: text + + TypeError: 'SequenceOfLong' object is not reversible + +Reverse iterators are slightly unusual, the `reversed()`_ function calls the object `__reversed__()`_ method if +available or falls back on using `__len__()`_ and `__getitem__()`_. + +`reversed()`_ acts like this python code: + +.. code-block:: python + + def reversed(obj: object): + if hasattr(obj, '__reversed__'): + yield from obj.__reversed__() + elif hasattr(obj, '__len__') and hasattr(obj, '__getitem__'): + i = len(obj) - 1 + while i >= 0: + yield obj[i] + i -= 1 + else: + raise TypeError(f'{type(object)} is not reversible') + + +To support this in C we can implement `__len__()`_ and `__getitem__()`_ using the `PySequenceMethods`_ method table. + +First the implementation of `__len__()`_ in C: + +.. code-block:: c + + static Py_ssize_t + SequenceOfLong_sq_length(PyObject *self) { + return ((SequenceOfLong *)self)->size; + } + +Then the implementation of `__getitem__()`_, note here that we support negative indexes and set and exception if the +index is out of range: + +.. code-block:: c + + static PyObject * + SequenceOfLong_sq_item(PyObject *self, Py_ssize_t index) { + Py_ssize_t my_index = index; + if (my_index < 0) { + my_index += SequenceOfLong_sq_length(self); + } + if (my_index > SequenceOfLong_sq_length(self)) { + PyErr_Format( + PyExc_IndexError, + "Index %ld is out of range for length %ld", + index, + SequenceOfLong_sq_length(self) + ); + return NULL; + } + return PyLong_FromLong(((SequenceOfLong *)self)->array_long[my_index]); + } + +Create `PySequenceMethods`_ method table: + +.. code-block:: c + + PySequenceMethods SequenceOfLong_sequence_methods = { + .sq_length = &SequenceOfLong_sq_length, + .sq_item = &SequenceOfLong_sq_item, + }; + +Add this method table into the type specification: + +.. code-block:: c + + static PyTypeObject SequenceOfLongType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* Other stuff. */ + .tp_dealloc = (destructor) SequenceOfLong_dealloc, + .tp_as_sequence = &SequenceOfLong_sequence_methods, + /* Other stuff. */ + }; + +And we can test it thus: + +.. code-block:: python + + def test_c_iterator_reversed(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = reversed(sequence) + assert list(result) == [4, 7, 1,] + + +.. index:: + single: Generators + +=========================== +Generators +=========================== + +Iterators are a requirement for `Generators `_, +the secret weapon in Pythons toolbox. +If you don't believe me then ask David Beazley who has done some very fine and informative +`presentations on Generators `_ + + +--------------------------------- +Our Iterator as a Generator +--------------------------------- + +Now we have an iterator we can write generator: + +.. code-block:: python + + def yield_from_an_iterator_times_two(iterator): + for value in iterator: + yield 2 * value + +And test it: + +.. code-block:: python + + def test_c_iterator_yield_forward(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + iterator = iter(sequence) + result = [] + for v in yield_from_an_iterator_times_two(iterator): + result.append(v) + assert result == [2, 14, 8] + +And create a `generator expression `_: + +.. code-block:: python + + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = sum(v * 4 for v in sequence) + assert result == 4 * (1 + 7 + 4) diff --git a/doc/sphinx/source/logging.rst b/doc/sphinx/source/logging.rst new file mode 100644 index 0000000..2815ac2 --- /dev/null +++ b/doc/sphinx/source/logging.rst @@ -0,0 +1,490 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + + +.. _chapter_logging_and_frames: +.. _chapter_logging_and_frames.logging: + +.. index:: + single: Logging + +================================= +Logging and Frames +================================= + +This chapter describes how to use the Python logging interface for C and how to access Python frames from C. + +---------------------- +Logging From C +---------------------- + +This presents a recipe for using the Python logging module in C. + +Many thanks to `nnathan `_ for starting this. + +We import the module and define C equivalent logging functions that are +compatible with the `*printf` family. + +Logging C Declarations +---------------------- + +First define the log levels, ``#define`` is used so the that switch case does not issue a compile time non-const error: + +.. code-block:: c + + #define PPY_SSIZE_T_CLEAN + #include + /* For va_start, va_end */ + #include + + /* logging levels defined by logging module + * From: https://docs.python.org/3/library/logging.html#logging-levels */ + #define LOGGING_DEBUG 10 + #define LOGGING_INFO 20 + #define LOGGING_WARNING 30 + #define LOGGING_ERROR 40 + #define LOGGING_CRITICAL 50 + #define LOGGING_EXCEPTION 60 + +Logging C Globals +---------------------- + +Then two globals, the first is the imported logging module, the next the current logger: + +.. code-block:: c + + /* This modules globals */ + static PyObject *g_logging_module = NULL; /* Initialise by PyInit_cLogging() below. */ + static PyObject *g_logger = NULL; + +Logging C Functions +---------------------- + +Now a function to get a logger object from the logging module: + +.. code-block:: c + + static PyObject *py_get_logger(char *logger_name) { + assert(g_logging_module); + PyObject *logger = NULL; + + logger = PyObject_CallMethod(g_logging_module, "getLogger", "s", logger_name); + if (logger == NULL) { + const char *err_msg = "failed to call logging.getLogger"; + PyErr_SetString(PyExc_RuntimeError, err_msg); + } + /* + fprintf(stdout, "%s()#%d logger=0x%p\n", __FUNCTION__, __LINE__, (void *)logger); + */ + return logger; + } + +Now the main interface to logging function: + +.. code-block:: c + + static PyObject * + py_log_msg(int log_level, char *printf_fmt, ...) { + assert(g_logger); + assert(!PyErr_Occurred()); + PyObject *log_msg = NULL; + PyObject *ret = NULL; + va_list fmt_args; + + va_start(fmt_args, printf_fmt); + log_msg = PyUnicode_FromFormatV(printf_fmt, fmt_args); + va_end(fmt_args); + + if (log_msg == NULL) { + /* fail. */ + ret = PyObject_CallMethod( + g_logger, + "critical", + "O", "Unable to create log message." + ); + } else { + /* call function depending on loglevel */ + switch (log_level) { + case LOGGING_DEBUG: + ret = PyObject_CallMethod(g_logger, "debug", "O", log_msg); + break; + case LOGGING_INFO: + ret = PyObject_CallMethod(g_logger, "info", "O", log_msg); + break; + case LOGGING_WARNING: + ret = PyObject_CallMethod(g_logger, "warning", "O", log_msg); + break; + case LOGGING_ERROR: + ret = PyObject_CallMethod(g_logger, "error", "O", log_msg); + break; + case LOGGING_CRITICAL: + ret = PyObject_CallMethod(g_logger, "critical", "O", log_msg); + break; + default: + ret = PyObject_CallMethod(g_logger, "critical", "O", log_msg); + break; + } + assert(!PyErr_Occurred()); + } + Py_DECREF(log_msg); + return ret; + } + +A function to set a log level: + +.. code-block:: c + + static PyObject * + py_log_set_level(PyObject *Py_UNUSED(module), PyObject *args) { + assert(g_logger); + PyObject *py_log_level; + + if (!PyArg_ParseTuple(args, "O", &py_log_level)) { + return NULL; + } + return PyObject_CallMethod(g_logger, "setLevel", "O", py_log_level); + } + +And the main function to log a message: + +.. code-block:: c + + static PyObject * + py_log_message(PyObject *Py_UNUSED(module), PyObject *args) { + int log_level; + char *message; + + if (!PyArg_ParseTuple(args, "iz", &log_level, &message)) { + return NULL; + } + return py_log_msg(log_level, "%s", message); + } + +cLogging Module +---------------------- + +Setup the module functions: + +.. code-block:: c + + static PyMethodDef logging_methods[] = { + /* ... */ + { + "py_log_set_level", + (PyCFunction) py_log_set_level, + METH_VARARGS, + "Set the logging level." + }, + { + "log", + (PyCFunction) py_log_message, + METH_VARARGS, + "Log a message." + }, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +The module definition: + +.. code-block:: c + + static PyModuleDef cLogging = { + PyModuleDef_HEAD_INIT, + .m_name = "cLogging", + .m_doc = "Logging mmodule.", + .m_size = -1, + .m_methods = logging_methods, + }; + +The module initialisation, this is where the logging module is imported with ``PyImport_ImportModule()``: + +.. code-block:: c + + PyMODINIT_FUNC PyInit_cLogging(void) { + PyObject *m = PyModule_Create(&cLogging); + if (! m) { + goto except; + } + g_logging_import = PyImport_ImportModule("logging"); + if (!g_logging_import) { + const char *err_msg = "failed to import 'logging'"; + PyErr_SetString(PyExc_ImportError, err_msg); + goto except; + } + g_logger = py_get_logger("cLogging"); + if (!g_logger) { + goto except; + } + /* Adding module globals */ + /* logging levels defined by logging module. + * Note: In Python logging FATAL = CRITICAL */ + if (PyModule_AddIntConstant(m, "INFO", LOGGING_INFO)) { + goto except; + } + if (PyModule_AddIntConstant(m, "WARNING", LOGGING_WARNING)) { + goto except; + } + if (PyModule_AddIntConstant(m, "ERROR", LOGGING_ERROR)) { + goto except; + } + if (PyModule_AddIntConstant(m, "FATAL", LOGGING_FATAL)) { + goto except; + } + if (PyModule_AddIntConstant(m, "CRITICAL", LOGGING_FATAL)) { + goto except; + } + if (PyModule_AddIntConstant(m, "DEBUG", LOGGING_DEBUG)) { + goto except; + } + if (PyModule_AddIntConstant(m, "EXCEPTION", LOGGING_EXCEPTION)) { + goto except; + } + + goto finally; + except: + /* abnormal cleanup */ + /* cleanup logger references */ + Py_XDECREF(g_logging_import); + Py_XDECREF(g_logger); + finally: + return m; + } + +Create in the ``setup.py``: + +.. code-block:: python + + Extension(name=f"{PACKAGE_NAME}.Logging.cLogging", + include_dirs=[], + sources=["src/cpy/Logging/cLogging.c", ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +And run ``python setup.py develop``. + +Using and Testing +----------------- + +Using From C +^^^^^^^^^^^^^ + +To simply use the interface defined in the above function, use it like the `printf` family of functions: + +.. code-block:: c + + py_log_msg(WARNING, "error code: %d", 10); + +Using From Python +^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + from cPyExtPatt.Logging import cLogging + + cLogging.log(cLogging.ERROR, "Test log message") + +There are various tests in ``tests/unit/test_c_logging.py``. +As pytest swallows logging messages there is a ``main()`` function in that script so you can run it from the command +line: + +.. code-block:: python + + def main(): + logger.setLevel(logging.DEBUG) + logger.info('main') + logger.warning('Test warning message XXXX') + logger.debug('Test debug message XXXX') + logger.info('_test_logging') + test_logging() + print() + print(cLogging) + print(dir(cLogging)) + print() + logger.info('cLogging.log():') + cLogging.py_log_set_level(10) + cLogging.log(cLogging.ERROR, "cLogging.log(): Test log message") + + return 0 + + + if __name__ == '__main__': + exit(main()) + +Here is an example output: + +.. code-block:: bash + + $python tests/unit/test_c_logging.py + 2025-03-07 11:49:23,994 7064 INFO main + 2025-03-07 11:49:23,994 7064 WARNING Test warning message XXXX + 2025-03-07 11:49:23,994 7064 DEBUG Test debug message XXXX + 2025-03-07 11:49:23,994 7064 INFO _test_logging + 2025-03-07 11:49:23,994 7064 WARNING Test warning message XXXX + 2025-03-07 11:49:23,994 7064 DEBUG Test debug message XXXX + + + ['CRITICAL', 'DEBUG', 'ERROR', 'EXCEPTION', 'INFO', 'WARNING', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__', 'c_file_line_function', 'log', 'py_file_line_function', 'py_log_set_level'] + + 2025-03-07 11:49:23,994 7064 INFO cLogging.log(): + 2025-03-07 11:49:23,994 7064 ERROR cLogging.log(): Test log message + + +.. _PyEval_GetFrame(): https://docs.python.org/3/c-api/reflection.html#c.PyEval_GetFrame +.. _PyFrameObject: https://docs.python.org/3/c-api/frame.html#c.PyFrameObject +.. _Frame API: https://docs.python.org/3/c-api/frame.html + +.. _chapter_logging_and_frames.frames: + +.. index:: + single: Frames + +---------------------- +Frames +---------------------- + +This describes how to extract the currently executing Python *frame* (or call stack if you wish). + +.. index:: + single: Frames; Python + +The Python Frame +---------------------- + +The crucial call is `PyEval_GetFrame()`_ that gets the current executing frame as a `PyFrameObject`_. +From Python 3.11 onwards the `PyFrameObject`_ has no public members instead there are many functions in the +`Frame API`_ to extract the data we need. Prior to that we could extract tha data directly. + +The example code is in ``src/cpy/Logging/cLogging.c``. + +.. code-block:: c + + /** + * Returns a tuple of the file, line and function of the current Python frame. + * Returns (None, 0, None) on failure. + * @param _unused_module + * @return PyObject *, a tuple of three values. + */ + static PyObject * + py_file_line_function(PyObject *Py_UNUSED(module)) { + const unsigned char *file_name = NULL; + const char *func_name = NULL; + int line_number = 0; + + PyFrameObject *frame = PyEval_GetFrame(); + if (frame) { + /* Get the file name. */ + #if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + /* See: + * https://docs.python.org/3.11/whatsnew/3.11.html#pyframeobject-3-11-hiding + */ + file_name = PyUnicode_1BYTE_DATA(PyFrame_GetCode(frame)->co_filename); + #else + file_name = PyUnicode_1BYTE_DATA(frame->f_code->co_filename); + #endif // PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + /* Get the line number. */ + line_number = PyFrame_GetLineNumber(frame); + /* Get the function name. */ + #if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + /* See: + * https://docs.python.org/3.11/whatsnew/3.11.html#pyframeobject-3-11-hiding + */ + func_name = (const char *) PyUnicode_1BYTE_DATA(PyFrame_GetCode(frame)->co_name); + #else + func_name = (const char *) PyUnicode_1BYTE_DATA(frame->f_code->co_name); + #endif // PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + } + /* Build the return value. */ + /* Use 'z' that makes Python None if the string is NULL. */ + return Py_BuildValue("ziz", file_name, line_number, func_name); + } + +And this function is registered thus: + +.. code-block:: c + + static PyMethodDef logging_methods[] = { + /* ... */ + { + "py_file_line_function", + (PyCFunction) py_file_line_function, + METH_NOARGS, + "Return the file, line and function name from the current Python frame." + }, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +And can be used thus (test code is in ``tests/unit/test_c_logging.py``): + +.. code-block:: python + + from cPyExtPatt.Logging import cLogging + + def test_py_file_line_function_file(): + file, _line, _function = cLogging.py_file_line_function() + assert file == __file__ + + + def test_py_file_line_function_line(): + _file, line, _function = cLogging.py_file_line_function() + assert line == 50 + + + def test_py_file_line_function_function(): + _file, _line, function = cLogging.py_file_line_function() + assert function == 'test_py_file_line_function_function' + +.. index:: + single: Frames; C + +The C "Frame" +----------------- + +It is also useful to be able to extract the C "frame" (actually the code location at compile time) +and present that as the same kind of Python tuple. + +A simple macro helps: + +.. code-block:: c + + #define C_FILE_LINE_FUNCTION Py_BuildValue("sis", __FILE__, __LINE__, __FUNCTION__) + +And is used thus: + +.. code-block:: c + + static PyObject * + c_file_line_function(PyObject *Py_UNUSED(module)) { + return C_FILE_LINE_FUNCTION; + } + +Teh function is registered thus: + +.. code-block:: c + + static PyMethodDef logging_methods[] = { + /* ... */ + { + "c_file_line_function", + (PyCFunction) c_file_line_function, + METH_NOARGS, + "Return the file, line and function name from the current C code." + }, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +And can be used thus (test code is in ``tests/unit/test_c_logging.py``): + +.. code-block:: python + + def test_c_file_line_function_file(): + file, line, function = cLogging.c_file_line_function() + assert file == 'src/cpy/Logging/cLogging.c' + assert line == 148 + assert function == 'c_file_line_function' diff --git a/doc/sphinx/source/memory_leaks.rst b/doc/sphinx/source/memory_leaks.rst new file mode 100644 index 0000000..dd37ae6 --- /dev/null +++ b/doc/sphinx/source/memory_leaks.rst @@ -0,0 +1,29 @@ +.. highlight:: python + :linenothreshold: 10 + +.. highlight:: c + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + + +.. _memory_leaks-label: + +.. index:: + single: Memory Leaks + +******************* +Memory Leaks +******************* + +This chapter describes some techniques for detecting and fixing memory leaks. +Much of this is taken from the `pymemtrace project `_ + +.. toctree:: + :maxdepth: 3 + + memory_leaks/introduction + memory_leaks/tools + memory_leaks/techniques + memory_leaks/pymemtrace diff --git a/doc/sphinx/source/memory_leaks/introduction.rst b/doc/sphinx/source/memory_leaks/introduction.rst new file mode 100644 index 0000000..026e3c8 --- /dev/null +++ b/doc/sphinx/source/memory_leaks/introduction.rst @@ -0,0 +1,276 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + +Introduction +==================== + +This describes tools and techniques that can identify memory leaks in long running Python programs. + +.. index:: + single: Memory Leaks; Is it a Leak? + +Is it a Leak? +------------------ + +Rising memory is not necessarily a leak, it can be internal data structures that grow naturally. +A common strategy for hash tables, arrays and the like is that when they reach capacity they reallocate themselves +into twice the original memory and this can look, superficially like a memory leak. + +Python data structures are not particularly efficient, an ``int`` is typically 24 bytes, a ``datetime`` 48 bytes and so on. + +A further source of 'leaks', or code that can mask memory loaks, are caches, in-memory databases and so on. + +.. index:: + single: Memory Leaks; Sources + +Sources of Leaks +------------------ + +Here is a non-exhaustive list in rough order of popularity: + +* Classic C/C++ leaks: + * ``malloc`` without corresponding ``free``. + * ``new`` without corresponding ``delete``. + * Static data structures that use ever increasing heap storage. +* Reference counting errors in C/C++ extensions used by Python. +* Bugs in C/C++ wrappers such as Cython or pybind11. +* Bugs in Python. + +.. index:: see: Memory Management; CPython Memory Management +.. index:: single: CPython Memory Management + +A Bit About (C)Python Memory Management +------------------------------------------ + +Python objects are allocated on the heap with their parent references on the stack. +When the stack unwinds the reference goes out of scope and, without any other action, the heap allocated would be leaked. +Python uses a couple of techniques to prevent this; reference counting and Garbage Collection. +Bear in mind that Python is quite old and the Garbage Collector reflects that [#]_. + +.. index:: single: CPython Memory Management; Reference Counts + +Reference Counts +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The primary technique that Python uses is reference counting, when an object is created it is given a reference count of 1. +When another object refers to it the reference count increases by one. +When an object goes out of scope that refers to the object the reference count is decremented by one. +When the reference count becomes zero the object is de-allocated and the memory can be re-used immediately. + +In C/C++ extensions you have to manage these reference counts manually and correctly. +An important point about reference counts in C/C++ extensions is that: + +* If they are too low the object might get de-allocated prematurely whilst there are still valid references. Then those + references might try to access the deleted object and that may, or may not, result in a segfault. +* If the reference count is incremented unnecessarily the object will never get de-allocated and there will be a memory leak. + +The latter is often regarded as the lesser of the two problems and the temptation is to err on the side of increasing +reference counts for 'safety'. +This swaps an easy to solve probem (segfault) for a harder to solve one (memory leak). + +You can find the reference count of any object by calling :py:func:`sys.getrefcount()` with the Python object as the argument. +The count is one higher than you might expect as it includes the (temporary) reference to the :py:func:`sys.getrefcount()`. + +Reference counting is always switched on in Python. + +.. index:: single: CPython Memory Management; Interned Objects + +.. note:: + + Some objects are *interned*, that is their reference count never goes to zero so that they are, in effect, permanent. + This is done for performance and includes most builtins and the integers -5 to 255. + + For example: + + .. code-block:: python + + >>> sys.getrefcount(None) + 14254 + >>> sys.getrefcount(0) + 2777 + >>> sys.getrefcount(400) + 2 + +.. index:: single: CPython Memory Management; Cyclic References + +Reference counts have one major problem, cyclic references. Consider this: + +.. code-block:: python + + class A: pass + a = A() + b = A() + a.next = b + b.next = a + +``a`` references ``b`` and ``b`` references ``a`` so you can not delete either without deleting the other. +To get round this problem Python uses a simple garbage collector. + +.. index:: single: CPython Memory Management; Garbage Collection + +Garbage Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The only job of Python's garbage collector (GC) is to discover unreachable objects that have cyclic references. +The Python garbage collector is fairly simple and rather old. +You can use the :py:mod:`gc` module to inspect and control the garbage collector. +The garbage collector can be switched off and this is often done in high performance systems. + +In particular: + +* The GC will not reclaim objects that are not tracked. + This includes many objects created in C/C++ extensions. + See :py:func:`gc.is_tracked` to see if an object is being tracked by the GC. +* The GC only looks at unreachable objects. +* The GC only deals with cyclic references. +* The GC is easily defeated, even inadvertently, for example if objects implement ``__del__``. +* A real restriction on the GC is due to C/C++ extensions. + An unreachable C/C++ object from Python code with a zero reference count can not be deleted as there is no way of + knowing if some C/C++ code might have a reference to it. + In Java this is easier as the VM controls the whole estate and can safely delete unreachable objects. + + +.. index:: single: CPython Memory Management; Memory Allocator + +The Big Picture +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here is a visualisation of memory allocators from top to bottom (from the Python source ``Objects/obmalloc.c``): + +.. code-block:: text + + _____ ______ ______ ________ + [ int ] [ dict ] [ list ] ... [ string ] Python core | + +3 | <----- Object-specific memory -----> | <-- Non-object memory --> | + _______________________________ | | + [ Python's object allocator ] | | + +2 | ####### Object memory ####### | <------ Internal buffers ------> | + ______________________________________________________________ | + [ Python's raw memory allocator (PyMem_ API) ] | + +1 | <----- Python memory (under PyMem manager's control) ------> | | + __________________________________________________________________ + [ Underlying general-purpose allocator (ex: C library malloc) ] + 0 | <------ Virtual memory allocated for the python process -------> | + + ========================================================================= + _______________________________________________________________________ + [ OS-specific Virtual Memory Manager (VMM) ] + -1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | + __________________________________ __________________________________ + [ ] [ ] + -2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | + + +Layer +2 is significant, it is the CPython's Object Allocator (``pymalloc``). + +.. index:: single: CPython Memory Management; pymalloc + +CPython's Object Allocator (``pymalloc``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Standard CPython uses an in-memory pool for small items (<=512 bytes) to reduce the cost of going to the OS for memory allocations. +One consequence of this is that small memory leaks will be hidden when observing the *overall* memory usage of a precess. +Another consequence is that tools such as Valgrind are rendered nearly useless for detecting memory leaks when the ``pymalloc`` is in use. +``pymalloc`` can be disabled with a special build of Python. +Requests >512 bytes are allocated without ``pymalloc`` and routed to the platform's allocator (usually the C ``malloc()`` function). + +A summary of ``pymalloc``: + +* ``pymalloc`` consists of a set of *Arena*'s. +* An *Arena* is a 256kB (262,144 bytes) chunk of memory divided up into *Pool*'s. +* A *Pool* is a chunk of memory the size of a OS page, usually 4096 bytes. +* A *Pool* is subdivided into *Block*'s which all have the same size for that Pool. +* A *Block* is memory sized between 8 and 512 (modulo 8). + +.. index:: single: CPython Memory Management; sys._debugmallocstats + +To understand this better try: + +.. code-block:: python + + import sys + sys._debugmallocstats() + +An you will get something like: + +.. code-block:: text + + Small block threshold = 512, in 64 size classes. + + class size num pools blocks in use avail blocks + ----- ---- --------- ------------- ------------ + 0 8 2 551 461 + 1 16 1 82 171 + 2 24 2 186 150 + ... + 62 504 10 73 7 + 63 512 19 132 1 + + # arenas allocated total = 95 + # arenas reclaimed = 46 + # arenas highwater mark = 49 + # arenas allocated current = 49 + 49 arenas * 262144 bytes/arena = 12,845,056 + + # bytes in allocated blocks = 12,129,080 + # bytes in available blocks = 174,784 + 59 unused pools * 4096 bytes = 241,664 + # bytes lost to pool headers = 147,696 + # bytes lost to quantization = 151,832 + # bytes lost to arena alignment = 0 + Total = 12,845,056 + + 14 free PyCFunctionObjects * 48 bytes each = 672 + 78 free PyDictObjects * 48 bytes each = 3,744 + 7 free PyFloatObjects * 24 bytes each = 168 + 3 free PyFrameObjects * 384 bytes each = 1,152 + 80 free PyListObjects * 40 bytes each = 3,200 + 17 free PyMethodObjects * 40 bytes each = 680 + 25 free 1-sized PyTupleObjects * 32 bytes each = 800 + 1446 free 2-sized PyTupleObjects * 40 bytes each = 57,840 + ... + 1 free 19-sized PyTupleObjects * 176 bytes each = 176 + +There are five sections: + +* The first line states the small object limit (512) and how this is divided: 512 / 8 = 64 'class's. + Each of these 'class's handle memory allocations of a specific size. +* The second section shows how many pools and blocks are in use for each 'class' (specific size of memory allocation). +* The third section is about *Arena*'s, there are currently 49 at 262,144 bytes each. +* The fourth section summarises the total memory usage, in particular the amount of memory consumed by the ``pymalloc`` administration. +* The fifth section is a summary of the memory consumed by particular Python type. + NOTE: This is not an exclusive list, many types such as ``int``, ``set`` are absent. + +In summary: + +.. code-block:: text + + - 49 Arenas of 256kB (262,144 bytes) is 12,845,056 in total. + - Each Arena is divided into 64 pools of 4096 bytes each, + thus 49 x 64 = 3136 pools (the sum of 'num pools') above. + - Each Pool of 4096 bytes is allocated a fixed size + between 8 and 512 bytes and divided by that into Blocks. + So there are between 512 x 8 byte blocks and 8 x 512 byte blocks in a Pool. + +See :ref:`memory_leaks.pymemtrace.debug_malloc_stats` for examples of ``pymemtrace.debug_malloc_stats`` +that can make this information much more useful. + +.. index:: single: CPython Memory Management; pymalloc deallocation + +Memory De-allocation +""""""""""""""""""""" + +* If the object is >512 bytes it is not under control of ``pymalloc`` and the memory is returned to the OS immediately. +* A *Pool* is free'd when all the blocks are empty. +* An *Arena* is free'd when all the *Pool*'s are empty. +* There is no attempt to reorganise ``pymalloc`` periodically reduce the memory use such as a copying garbage collector might do. + +This means that pools and arenas can exist for a very long time. + +.. Example footnote [#]_. + +.. rubric:: Footnotes + +.. [#] This chapter discusses Pythons memory management system however the design of that may change over the range of + Python versions that this project covers. + Therefore some descriptions may be inaccurate or obsolete depending on the Python version under consideration. diff --git a/doc/sphinx/source/memory_leaks/pymemtrace.rst b/doc/sphinx/source/memory_leaks/pymemtrace.rst new file mode 100644 index 0000000..07e80bd --- /dev/null +++ b/doc/sphinx/source/memory_leaks/pymemtrace.rst @@ -0,0 +1,674 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + + +.. _memory-leaks.pymemtrace: + +.. index:: + single: pymemtrace + single: Memory Leaks; pymemtrace + +============================= +``pymemtrace`` +============================= + +My ``pymemtrace`` project contains a number of tools that help detect memory usage and leaks. +The documentation contains advice on handling memory leaks. + +* On PyPi: ``_ +* Project: ``_ +* Documentation: ``_ + +Here is the introduction to that project: + +``pymemtrace`` provides tools for tracking and understanding Python memory usage at different levels, at different +granularities and with different runtime costs. + +Full documentation: https://pymemtrace.readthedocs.io + +.. _DTrace examples: https://pymemtrace.readthedocs.io/en/latest/examples/dtrace.html +.. _technical note on DTrace: https://pymemtrace.readthedocs.io/en/latest/tech_notes/dtrace.html#tech-notes-dtrace + +.. index:: + single: pymemtrace; Tools + +pymemtrace Tools +====================== + +The tools provided by ``pymemtrace``: + +* ``process`` is a very lightweight way of logging the total memory usage at regular time intervals. + It can plot memory over time with plotting programs such as ``gnuplot``. + See `some process examples `_ +* ``cPyMemTrace`` is a memory tracer written in C that can report total memory usage for every function call/return for + both C and Python sections. + See some `cPyMemTrace examples `_ + and a `technical note on cPyMemTrace `_. +* DTrace: Here are a number of D scripts that can trace the low level ``malloc()`` and ``free()`` system calls and + report how much memory was allocated and by whom. + See some `DTrace examples`_ and a `technical note on DTrace`_. +* ``trace_malloc`` is a convenience wrapper around the Python standard library `tracemalloc` module. + This can report Python memory usage by module and line compensating for the cost of ``tracemalloc``. + This can take memory snapshots before and after code blocks and show the change on memory caused by that code. + See some `trace_malloc examples `_ +* ``debug_malloc_stats`` is a wrapper around the ``sys._debugmallocstats`` function that can take snapshots of + memory before and after code execution and report the significant differences of the Python small object allocator. + See some `debug_malloc_stats examples `_ + + +.. index:: + single: pymemtrace; Tool Characteristics + +Tool Characteristics +====================== + +Each tool can be characterised by: + +- *Memory Granularity*: In how much detail is a memory change is observed. + An example of *coarse* memory granularity is measuring the + `Resident Set Size `_ which is normally in chunks of 4096 bytes. + An example of *fine* memory granularity is recording every ``malloc()`` and ``free()``. +- *Execution Granularity*: In how much code detail is the memory change observed. + An example of *coarse* execution granularity is measuring the memory usage every second. + An example of *fine* execution granularity is recording the memory usage every Python line. +- *Memory Cost*: How much extra memory the tool needs. +- *Execution Cost*: How much the execution time is increased. + +Clearly there are trade-offs between these depending on the problem you are trying to solve. + +.. list-table:: **Tool Characteristics** + :widths: 30 30 30 20 20 + :header-rows: 1 + + * - Tool + - Memory Granularity + - Execution Granularity + - Memory Cost + - Execution Cost + * - ``process`` + - RSS (total Python and C memory). + - Regular time intervals. + - Near zero. + - Near zero. + * - ``cPyMemTrace`` + - RSS (total Python and C memory). + - Per Python line, Python function and C function call. + - Near zero. + - x10 to x20. + * - DTrace + - Every ``malloc()`` and ``free()``. + - Per function call and return. + - Minimal. + - x100. + * - ``trace_malloc`` + - Every Python object. + - Per Python line, per function call. + - Significant but compensated. + - x900 for small objects, x6 for large objects. + * - ``debug_malloc_stats`` + - Python memory pool. + - Snapshots the CPython memory pool either side of a block of code. + - Minimal. + - x2000+ for small objects, x12 for large objects. + +Licence +------- + +Python memory tracing. + +* Free software: MIT license +* Documentation: https://pymemtrace.readthedocs.io. +* Project: https://github.com/paulross/pymemtrace. + +Credits +------- + +Phil Smith (AHL) with whom a casual lunch time chat lead to the creation of an earlier, but quite different +implementation, of ``cPyMemTrace`` in pure Python. + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage + + +.. index:: + single: pymemtrace; process + +.. _chapter_memory_leaks.pymemtrace.proces: + +``pymemtrace`` Process +====================== + +`pymemtrac's process `_ is an ultralight weight tool +for monitoring the memory usage of a process at regular intervals. + +``process.log_process`` provides a context manager that launches a separate thread that logs the memory usage in JSON +of the current process at regular intervals (the CLI version can monitor any user specified process). +The log format is designed so that the data can be easily extracted using, say, regular expressions. + +Here is an example that creates randomly sized large strings: + +.. code-block:: python + + """ + Example of using process that logs process data to the current log. + """ + import logging + import random + import sys + import time + + from pymemtrace import process + + logger = logging.getLogger(__file__) + + def main() -> int: + logging.basicConfig( + level=logging.INFO, + format= ( + '%(asctime)s - %(filename)s#%(lineno)d - %(process)5d' + ' - (%(threadName)-10s) - %(levelname)-8s - %(message)s' + ), + ) + logger.info('Demonstration of logging a process') + # Log process data to the log file every 0.5 seconds. + with process.log_process(interval=0.5, log_level=logger.getEffectiveLevel()): + for i in range(8): + size = random.randint(128, 128 + 256) * 1024 ** 2 + # Add a message to report in the next process write. + process.add_message_to_queue(f'String of {size:,d} bytes') + s = ' ' * size + time.sleep(0.75 + random.random()) + del s + time.sleep(0.25 + random.random() / 2) + return 0 + + + if __name__ == '__main__': + sys.exit(main()) + +Might give this, although verbose, it is quite searchable: + +.. code-block:: text + + $ python3.12 ex_process.py + 2025-02-12 14:16:58,675 - ex_process.py#19 - 10193 - (MainThread) - INFO - Demonstration of logging a process + 2025-02-12 14:16:58,676 - process.py#289 - 10193 - (ProcMon ) - INFO - ProcessLoggingThread-JSON-START {"timestamp": "2025-02-12 14:16:58.676195", "memory_info": {"rss": 18067456, "vms": 34990526464, "pfaults": 6963, "pageins": 1369}, "cpu_times": {"user": 0.340946528, "system": 0.991057664, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 5.407800197601318, "pid": 10193} + 2025-02-12 14:16:59,180 - process.py#293 - 10193 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2025-02-12 14:16:59.180476", "memory_info": {"rss": 199512064, "vms": 35171934208, "pfaults": 51261, "pageins": 1374}, "cpu_times": {"user": 0.379827552, "system": 1.031979648, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 5.912204027175903, "pid": 10193, "label": "String of 181,403,648 bytes"} + 2025-02-12 14:16:59,682 - process.py#289 - 10193 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2025-02-12 14:16:59.681947", "memory_info": {"rss": 18104320, "vms": 34990526464, "pfaults": 51262, "pageins": 1374}, "cpu_times": {"user": 0.380316928, "system": 1.047401792, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 6.413706064224243, "pid": 10193} + ... + 2025-02-12 14:17:12,312 - process.py#289 - 10193 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2025-02-12 14:17:12.312343", "memory_info": {"rss": 247758848, "vms": 35220168704, "pfaults": 508755, "pageins": 1374}, "cpu_times": {"user": 0.820292992, "system": 1.639239552, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 19.044106006622314, "pid": 10193} + 2025-02-12 14:17:12,763 - process.py#289 - 10193 - (MainThread) - INFO - ProcessLoggingThread-JSON-STOP {"timestamp": "2025-02-12 14:17:12.762896", "memory_info": {"rss": 18116608, "vms": 34990526464, "pfaults": 508756, "pageins": 1374}, "cpu_times": {"user": 0.820827264, "system": 1.663195264, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 19.49466300010681, "pid": 10193} + + Process finished with exit code 0 + +Here is the memory data from one line in more detail. + +.. code-block:: json + + { + "timestamp": "2025-02-12 14:17:05.719867", + "memory_info": { + "rss": 264527872, + "vms": 35236945920, + "pfaults": 342856, + "pageins": 1374 + }, + "cpu_times": { + "user": 0.658989888, + "system": 1.418129152, + "children_user": 0.0, + "children_system": 0.0 + }, + "elapsed_time": 12.45150899887085, + "pid": 10193, + "label": "String of 246,415,360 bytes" + } + +Of interest is the line: + +.. code-block:: python + + process.add_message_to_queue(f'String of {size:,d} bytes') + +Which injects a message into the log output and that has the JSON key of "label". + + +``pymemtrace.process`` provides and number of ways of tabulating and plotting this data that gives a clearer picture, +at a high level, of what is happening to the process memory. +Some can be seen `_ + +.. index:: + single: pymemtrace; cPyMemTrace + single: cPyMemTrace + +``pymemtrace`` cPyMemTrace +========================== + +``cPyMemTrace`` is a Python profiler written in 'C' that records the +`Resident Set Size `_ +for every Python and C call and return. + +``cPyMemTrace`` writes this data to a log file to the current working directory with a name of the following form, +it combines: + +- ``YYYYMMDD`` The date. +- ``_HHMMSS`` The time. +- ``_PID`` The process ID. +- ``_P`` or ``_T`` depending on whether it is a profile function or a trace function. +- ``_n`` where n is the stack depth of the current profile or trace function as multiple nested profile or trace + functions are allowed. +- The Python version such as ``PY3.13.0b3``. +- ``.log``. + +For example ``"20241107_195847_62264_P_2_PY3.13.0b3.log"`` + +Logging Changes in RSS +-------------------------------- + +Here is a simple example: + +.. code-block:: python + + from pymemtrace import cPyMemTrace + + def create_string(l: int) -> str: + return ' ' * l + + with cPyMemTrace.Profile(): + l = [] + for i in range(8): + l.append(create_string(1024**2)) + while len(l): + l.pop() + +This produces a log file in the current working directory. +For brevity the log file does not show every profile or trace event but only those when the RSS changes by some +threshold. +By default this threshold is the system page size (typically 4096 bytes) [#]_. + +Here is an example output: + +.. code-block:: text + + Event dEvent Clock What File #line Function RSS dRSS + NEXT: 0 +0 0.066718 CALL test.py # 9 create_string 9101312 9101312 + NEXT: 1 +1 0.067265 RETURN test.py # 10 create_string 10153984 1052672 + PREV: 4 +3 0.067285 CALL test.py # 9 create_string 10153984 0 + NEXT: 5 +4 0.067777 RETURN test.py # 10 create_string 11206656 1052672 + PREV: 8 +3 0.067787 CALL test.py # 9 create_string 11206656 0 + NEXT: 9 +4 0.068356 RETURN test.py # 10 create_string 12259328 1052672 + PREV: 12 +3 0.068367 CALL test.py # 9 create_string 12259328 0 + NEXT: 13 +4 0.068944 RETURN test.py # 10 create_string 13312000 1052672 + PREV: 16 +3 0.068954 CALL test.py # 9 create_string 13312000 0 + NEXT: 17 +4 0.069518 RETURN test.py # 10 create_string 14364672 1052672 + PREV: 20 +3 0.069534 CALL test.py # 9 create_string 14364672 0 + NEXT: 21 +4 0.070101 RETURN test.py # 10 create_string 15417344 1052672 + PREV: 24 +3 0.070120 CALL test.py # 9 create_string 15417344 0 + NEXT: 25 +4 0.070663 RETURN test.py # 10 create_string 16470016 1052672 + PREV: 28 +3 0.070677 CALL test.py # 9 create_string 16470016 0 + NEXT: 29 +4 0.071211 RETURN test.py # 10 create_string 17522688 1052672 + +So in this example events 2 and 3 are omitted as there is no change in the RSS. +Events 4 and 5 are included as there is a change in the RSS between them. + +Logging Every Event +-------------------------------- + +If all events are needed then change the constructor argument to 0: + +.. code-block:: python + + with cPyMemTrace.Profile(0): + # As before + +And the log file looks like this: + +.. code-block:: text + + Event dEvent Clock What File #line Function RSS dRSS + NEXT: 0 +0 0.079408 CALL test.py # 9 create_string 9105408 9105408 + NEXT: 1 +1 0.079987 RETURN test.py # 10 create_string 10158080 1052672 + NEXT: 2 +1 0.079994 C_CALL test.py # 64 append 10158080 0 + NEXT: 3 +1 0.079998 C_RETURN test.py # 64 append 10158080 0 + NEXT: 4 +1 0.080003 CALL test.py # 9 create_string 10158080 0 + NEXT: 5 +1 0.080682 RETURN test.py # 10 create_string 11210752 1052672 + NEXT: 6 +1 0.080693 C_CALL test.py # 64 append 11210752 0 + NEXT: 7 +1 0.080698 C_RETURN test.py # 64 append 11210752 0 + NEXT: 8 +1 0.080704 CALL test.py # 9 create_string 11210752 0 + NEXT: 9 +1 0.081414 RETURN test.py # 10 create_string 12263424 1052672 + NEXT: 10 +1 0.081424 C_CALL test.py # 64 append 12263424 0 + NEXT: 11 +1 0.081429 C_RETURN test.py # 64 append 12263424 0 + NEXT: 12 +1 0.081434 CALL test.py # 9 create_string 12263424 0 + NEXT: 13 +1 0.081993 RETURN test.py # 10 create_string 13316096 1052672 + NEXT: 14 +1 0.081998 C_CALL test.py # 64 append 13316096 0 + ... + NEXT: 59 +1 0.084531 C_RETURN test.py # 66 pop 17526784 0 + NEXT: 60 +1 0.084535 C_CALL test.py # 65 len 17526784 0 + NEXT: 61 +1 0.084539 C_RETURN test.py # 65 len 17526784 0 + NEXT: 62 +1 0.084541 C_CALL test.py # 66 pop 17526784 0 + NEXT: 63 +1 0.084561 C_RETURN test.py # 66 pop 17526784 0 + NEXT: 64 +1 0.084566 C_CALL test.py # 65 len 17526784 0 + NEXT: 65 +1 0.084568 C_RETURN test.py # 65 len 17526784 0 + +There is some discussion about the performance of ``cPyMemTrace`` here in the +`technical note on cPyMemTrace `_. +and some more +`cPyMemTrace code examples here `_ + +Tracers can be nested such as this and each level gets logged to its own file, for example: + +.. code-block:: python + + from pymemtrace import cPyMemTrace + + with cPyMemTrace.Profile(): + # Do stuff at stack level 0 + # This gets logged into, say: "20241107_195847_62264_P_0_PY3.13.0b3.log" + with cPyMemTrace.Profile(): + # Do stuff at stack level 0 + # This gets logged into, say: "20241107_195847_62264_P_1_PY3.13.0b3.log" + pass + # Do stuff at stack level 0 + # This gets logged again into: "20241107_195847_62264_P_0_PY3.13.0b3.log" + + + + +.. index:: + single: pymemtrace; DTrace + single: DTrace + +``pymemtrace`` and DTrace +========================= + +With an OS that supports DTrace (for example Mac OS X) ``pymemtrace`` provides some D scripts that support memory +profiling. +DTrace is an extremely powerful tool that can produce an enormous amount of detailed information on memory +allocations and de-allocations. + +This is beyond the scope of *this* document however some examples are shown in `DTrace examples`_. +There is some discussion about the performance of ``DTrace`` here in the `technical note on DTrace`_ +and some more +`DTrace code examples here `_. + +.. index:: + single: pymemtrace; tracemalloc + single: tracemalloc + +.. _memory_leaks.pymemtrace.trace_malloc: + +``pymemtrace`` trace_malloc +================================= + +Python has the :py:mod:`tracemalloc` module +(`documentation `_) +that can provide the following information: + +- Trace where an object was allocated. +- Statistics on allocated memory blocks per filename and per line number: + total size, number and average size of allocated memory blocks. +- Compute the differences between two snapshots to detect memory leaks. + +However the :py:mod:`tracemalloc` also consumes memory so this can conceal what is really going on. + +``pymemtrace.trace_malloc`` contains some utility wrappers around the :py:mod:`tracemalloc` module and +the can compensate for the memory used by :py:mod:`tracemalloc` module. + +Using ``trace_malloc`` Directly +---------------------------------------- + +Adding 1Mb Strings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here is an example of adding 1Mb strings to a list under the watchful eye of :py:class:`trace_malloc.TraceMalloc`: + +.. code-block:: python + + from pymemtrace import trace_malloc + + list_of_strings = [] + print(f'example_trace_malloc_for_documentation()') + with trace_malloc.TraceMalloc('filename') as tm: + for i in range(8): + list_of_strings.append(' ' * 1024**2) + print(f' tm.memory_start={tm.memory_start}') + print(f'tm.memory_finish={tm.memory_finish}') + print(f' tm.diff={tm.diff}') + for stat in tm.statistics: + print(stat) + +Typical output is: + +.. code-block:: text + + example_trace_malloc_for_documentation() + tm.memory_start=13072 + tm.memory_finish=13800 + tm.diff=8388692 + pymemtrace/examples/ex_trace_malloc.py:0: size=8194 KiB (+8193 KiB), count=16 (+10), average=512 KiB + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/tracemalloc.py:0: size=6464 B (+504 B), count=39 (+10), average=166 B + Documents/workspace/pymemtrace/pymemtrace/trace_malloc.py:0: size=3076 B (-468 B), count=10 (-1), average=308 B + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py:0: size=16.3 KiB (-128 B), count=49 (-2), average=340 B + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/abc.py:0: size=3169 B (+0 B), count=30 (+0), average=106 B + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/posixpath.py:0: size=480 B (+0 B), count=1 (+0), average=480 B + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/threading.py:0: size=168 B (+0 B), count=2 (+0), average=84 B + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/_weakrefset.py:0: size=72 B (+0 B), count=1 (+0), average=72 B + + +To eliminate the lines that is caused by ``tracemalloc`` itself change the last two lines to: + +.. code-block:: python + + for stat in tm.net_statistics: + print(stat) + +Which removes the line: + +.. code-block:: text + + /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/tracemalloc.py:0: size=6464 B (+504 B), count=39 (+10), average=166 B + +Using ``trace_malloc`` as a Decorator +---------------------------------------- + +``trace_malloc`` provides a function decorator that can log the tracemalloc memory usage caused by execution a function. +For example: + +.. code-block:: python + + from pymemtrace import trace_malloc + + @trace_malloc.trace_malloc_log(logging.INFO) + def example_decorator_for_documentation(list_of_strings): + for i in range(8): + list_of_strings.append(create_string(1024**2)) + + list_of_strings = [] + example_decorator_for_documentation(list_of_strings) + +Would log something like the following: + +.. code-block:: text + + 2025-02-13 11:37:39,194 - trace_malloc.py#87 - 10121 - (MainThread) - INFO - TraceMalloc memory delta: 8,389,548 for "example_decorator_for_documentation()" + +Here are more `examples `_ +and some more +`trace_malloc code examples here `_ + + +.. index:: + single: pymemtrace; Debug Malloc Stats + single: Debug Malloc Stats + single: sys._debugmallocstats() + +.. _memory_leaks.pymemtrace.debug_malloc_stats: + +``pymemtrace`` Debug Malloc Stats +================================= + +CPython has the function the :py:func:`sys._debugmallocstats()` that can dump the status of the Python small object +memory allocator. +For example: + +.. code-block:: python + + Python 3.13.0 (v3.13.0:60403a5409f, Oct 7 2024, 00:37:40) [Clang 15.0.0 (clang-1500.3.9.4)] on darwin + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys._debugmallocstats() + Small block threshold = 512, in 32 size classes. + + class size num pools blocks in use avail blocks + ----- ---- --------- ------------- ------------ + 0 16 1 90 931 + 1 32 2 830 190 + 2 48 9 2916 144 + 3 64 48 11539 701 + 4 80 30 6076 44 + 5 96 6 999 21 + 6 112 4 454 126 + 7 128 8 943 73 + 8 144 2 188 38 + 9 160 19 1915 23 + 10 176 2 126 58 + 11 192 2 100 70 + 12 208 5 342 48 + 13 224 4 277 11 + 14 240 5 271 69 + 15 256 4 189 63 + 16 272 3 144 36 + 17 288 3 125 43 + 18 304 2 88 18 + 19 320 2 78 24 + 20 336 2 54 42 + 21 352 2 50 42 + 22 368 2 54 34 + 23 384 2 49 35 + 24 400 5 171 29 + 25 416 2 40 38 + 26 432 1 26 11 + 27 448 1 28 8 + 28 464 1 34 1 + 29 480 1 28 6 + 30 496 1 26 6 + 31 512 2 41 21 + + # arenas allocated total = 3 + # arenas reclaimed = 0 + # arenas highwater mark = 3 + # arenas allocated current = 3 + 3 arenas * 1048576 bytes/arena = 3,145,728 + + # bytes in allocated blocks = 2,654,688 + # bytes in available blocks = 322,672 + 6 unused pools * 16384 bytes = 98,304 + # bytes lost to pool headers = 8,784 + # bytes lost to quantization = 12,128 + # bytes lost to arena alignment = 49,152 + Total = 3,145,728 + + arena map counts + # arena map mid nodes = 1 + # arena map bot nodes = 1 + + # bytes lost to arena map root = 262,144 + # bytes lost to arena map mid = 262,144 + # bytes lost to arena map bot = 131,072 + Total = 655,360 + + 55 free PyDictObjects * 48 bytes each = 2,640 + 6 free PyFloatObjects * 24 bytes each = 144 + 80 free PyListObjects * 40 bytes each = 3,200 + 3 free 1-sized PyTupleObjects * 32 bytes each = 96 + 814 free 2-sized PyTupleObjects * 40 bytes each = 32,560 + 58 free 3-sized PyTupleObjects * 48 bytes each = 2,784 + 20 free 4-sized PyTupleObjects * 56 bytes each = 1,120 + 6 free 5-sized PyTupleObjects * 64 bytes each = 384 + 3 free 6-sized PyTupleObjects * 72 bytes each = 216 + 1 free 7-sized PyTupleObjects * 80 bytes each = 80 + 3 free 8-sized PyTupleObjects * 88 bytes each = 264 + 2 free 9-sized PyTupleObjects * 96 bytes each = 192 + 1 free 10-sized PyTupleObjects * 104 bytes each = 104 + 1 free 11-sized PyTupleObjects * 112 bytes each = 112 + 0 free 12-sized PyTupleObjects * 120 bytes each = 0 + 3 free 13-sized PyTupleObjects * 128 bytes each = 384 + 0 free 14-sized PyTupleObjects * 136 bytes each = 0 + 4 free 15-sized PyTupleObjects * 144 bytes each = 576 + 1 free 16-sized PyTupleObjects * 152 bytes each = 152 + 1 free 17-sized PyTupleObjects * 160 bytes each = 160 + 1 free 18-sized PyTupleObjects * 168 bytes each = 168 + 0 free 19-sized PyTupleObjects * 176 bytes each = 0 + 2 free 20-sized PyTupleObjects * 184 bytes each = 368 + >>> + +The drawback of this is that you really want to see the before and after snapshot when a particular operation is +performed and this means comparing quite verbose output. + +``pymemtrace`` has a module ``debug_malloc_stats`` that can provide is a wrapper around the +:py:func:`sys._debugmallocstats` function which take snapshots of +memory before and after code execution and report the significant differences of the Python small object allocator. +It uses a text parser to show only the differences between before and after. +For example: + +.. code-block:: python + + from pymemtrace import debug_malloc_stats + + print(f'example_debug_malloc_stats_for_documentation()') + list_of_strings = [] + with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: + for i in range(1, 9): + list_of_strings.append(' ' * (i * 8)) + print(f'DiffSysDebugMallocStats.diff():') + print(f'{malloc_diff.diff()}') + +The output is: + +.. code-block:: text + + example_debug_malloc_stats_for_documentation() + DiffSysDebugMallocStats.diff(): + class size num pools blocks in use avail blocks + ----- ---- --------- ------------- ------------ + 1 32 +1 +52 +74 + 2 48 +0 +17 -17 + 3 64 +0 +33 -33 + 4 80 +1 +51 -1 + 5 96 +2 +34 +50 + 6 112 +0 +2 -2 + 7 128 +0 +1 -1 + 10 176 +0 +1 -1 + 12 208 +0 +1 -1 + 17 288 +0 +1 -1 + 18 304 +0 +2 -2 + 25 416 +0 +3 -3 + 26 432 +0 +3 -3 + 27 448 +0 +3 -3 + 29 480 +0 +3 -3 + 30 496 +0 +1 -1 + 31 512 +0 +1 -1 + + # bytes in allocated blocks = +19,904 + # bytes in available blocks = -3,808 + -4 unused pools * 4096 bytes = -16,384 + # bytes lost to pool headers = +192 + # bytes lost to quantization = +96 + + -1 free 1-sized PyTupleObjects * 32 bytes each = -32 + +1 free 5-sized PyTupleObjects * 64 bytes each = +64 + +2 free PyDictObjects * 48 bytes each = +96 + -2 free PyListObjects * 40 bytes each = -80 + +1 free PyMethodObjects * 48 bytes each = +48 + +There are more examples in https://pymemtrace.readthedocs.io/en/latest/examples/debug_malloc_stats.html +and some more +`debug_malloc_stats code examples here `_ + +.. Example footnote [#]_. + +.. rubric:: Footnotes + +.. [#] This is obtained from ``int getpagesize(void);`` in ``#include ``. diff --git a/doc/sphinx/source/memory_leaks/techniques.rst b/doc/sphinx/source/memory_leaks/techniques.rst new file mode 100644 index 0000000..111ee38 --- /dev/null +++ b/doc/sphinx/source/memory_leaks/techniques.rst @@ -0,0 +1,44 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + +.. index:: single: Memory Leaks; Techniques + +Techniques +==================================== + +This describes some of the techniques I have found useful. +Bear in mind: + +* Tracking down memory leaks can take a long, long time. +* Every memory leak is its own special little snowflake! + So what works will be situation specific. + +High Level +------------------ + +It is worth spending a fair bit of time at high level before diving into the code since: + +* Working at high level is relatively cheap. +* It is usually non-invasive. +* It will quickly find out the *scale* of the problem. +* It will quickly find out the *repeatability* of the problem. +* You should be able to create the test that shows that the leak is firstly not fixed, then fixed. + +At the end of this you should be able to state: + +* The *frequency* of the memory leak. +* The *severity* of the memory leak. + +Relevant quote: **"Time spent on reconnaissance is seldom wasted."** + + +Using Platform Tools +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The high level investigation will usually concentrate on using platform tools such as builtin memory management tools or +Python tools such as ``pymentrace``'s :ref:`chapter_memory_leaks.pymemtrace.proces` or ``psutil`` will prove useful. + +Specific Tricks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +TODO: Finish this. diff --git a/doc/sphinx/source/memory_leaks/tools.rst b/doc/sphinx/source/memory_leaks/tools.rst new file mode 100644 index 0000000..cd34ada --- /dev/null +++ b/doc/sphinx/source/memory_leaks/tools.rst @@ -0,0 +1,275 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + +.. index:: single: Memory Leaks; Tools + +Tools for Detecting Memory Leaks +==================================== + +Tools for analysing memory can be characterised by: + +.. list-table:: Tool Characteristics + :widths: 25 75 + :header-rows: 1 + + * - Characteristic + - Description + * - **Availability** + - Does it come with the platform? + Is it within Python or the standard library? + Does it need third party library installation or requires a special Python build of some sort? + * - **Memory Granularity** + - How detailed is the memory measurement? + Somewhere between every ``malloc`` or the overall memory usage as seen by the OS. + * - **Execution Granularity** + - How detailed is the execution measurement? Per line, per function, for Python or C code? + * - **Memory Cost** + - What is the extra memory consumption is introduced by using this tool? + * - **Execution Cost** + - What is the extra runtime introduced by using this tool? + * - **Developer Cost** + - How hard it is to use the tool? + +Each tool makes trade offs between each of these characteristics. + +.. index:: single: Memory Leaks; Platform Tools + +Platform Tools +------------------ + +These tools come ready with the platform. They give a good overall picture of the memory usage. + +=========================== ==================================================================================================== +Characteristic Description +=========================== ==================================================================================================== +**Availability** Always. +**Memory Granularity** Usually the total memory usage by the process. +**Execution Granularity** Generally periodic at low frequency, typically of the order of seconds. +**Memory Cost** Usually none. +**Execution Cost** Usually none. +**Developer Cost** Easy. +=========================== ==================================================================================================== + +Here are some tools available on differenct platforms: + +Windows +^^^^^^^^^^^^^^^^^^^ + +A weakness of Windows, especially in corporate environments, is that the OS is usually severely locked down. +At best this usually vastly extends the time it takes to find a leak, at worst this hopelessly limits the tools that can +be installed or run on the platform. +In this case some leaks can never be found and fixed. + +Windows Task Manager +"""""""""""""""""""""""""""" + +This is the basic tool for reviewing process memory. +The columns to monitor are "Working Set (Memory)" which broadly corresponds to the Unix +`Resident Set Size (RSS) `_ . +The Sysinternals ``procexp`` (see below) is a more sophisticated version. + +``perfmon.exe`` +"""""""""""""""""""""""""""" + +This is a Microsoft tool for logging performance and plotting it in real time. +It is quite capable but a little fiddly to set up. +The third party Python library ``psutil`` is a useful alternative and the ``pymentrace.procces`` also provides memory +plotting of arbitrary processes: :ref:`chapter_memory_leaks.pymemtrace.proces` using ``gnuplot``. + +Sysinternals Suite +"""""""""""""""""""""""""""" + +The outstanding `Windows Sysinternals tools `_ are a wonderful +collection of tools and are essential for debugging any Windows application. + +Linux +^^^^^^^^^^^^^^^^^^^ + +TODO: Finish this. + +``/proc/`` +""""""""""""""""""""" + +The ``/proc/`` filesystem is full of good stuff. + +Valgrind +""""""""""""""""" + +`Valgrind `_ is essential on any Linux development platform. +There is a tutorial `here `_ for +building and using Python with Valgrind. + +eBPF +""""""""""""""""" + +The next big thing after DTrace is `eBPF `_. +Truly awesome. + + +Mac OS X +^^^^^^^^^^^^^^^^^^^ + +Tools such as ``vmmap``, ``heap``, ``leaks``, ``malloc_history``, ``vm_stat`` can all help. +See the man pages for further information. + +Some useful information for memory tools from +`Apple `_ + + +DTrace +""""""""""""""""""" + +Mac OS X is DTrace aware, this needs a special build of Python, here is an +`introduction `_ that takes you through building and using a DTrace aware version +of Python on Mac OS X. + +Some examples of using DTrace with +`pymemtrace `_. + +.. index:: single: Memory Leaks; Python Tools + +Python Tools +------------------ + +Modules from the Standard Library +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ==================================================================================================== +Characteristic Description +=========================== ==================================================================================================== +**Availability** Always. +**Memory Granularity** Various. +**Execution Granularity** Various. +**Memory Cost** Usually none. +**Execution Cost** Usually none. +**Developer Cost** Straightforward. +=========================== ==================================================================================================== + +``sys`` +""""""""""""""""""""" + +The :py:mod:`sys` has a number of useful functions, mostly CPython specific. + +.. Sigh. Links do not work in list tables such as ``_ + + +.. list-table:: ``sys`` Tools + :widths: 40 40 40 + :header-rows: 1 + + * - Tool + - Description + - Notes + * - ``getallocatedblocks()`` + - Returns the number of + `allocated blocks `_, regardless of size. + - This has no information about the size of any block. + CPython only. + Implemented in ``Objects/obmalloc.c`` as ``_Py_GetAllocatedBlocks``. + As implemented in Python 3.9 this returns the total reference count of every *pool* in every *arena*. + * - ``getrefcount(object)`` + - Returns the `reference count `_ of an object. + - This is increased by one for the duration of the call. + * - ``getsizeof(object)`` + - Returns the `size of an object `_ in bytes. + - Builtin objects will return correct results. + Others are implementation specific. + User defined objects can implement ``__sizeof__`` which will be called if available. + * - ``_debugmallocstats(object)`` + - Prints the state of the + `Python Memory Allocator `_ + ``pymalloc`` to stderr. + ``pymentrace``'s :ref:`memory_leaks.pymemtrace.debug_malloc_stats` is a very useful wrapper around + :py:func:`sys._debugmallocstats` which can report changes to Python's small object allocator. + - See :ref:`memory_leaks.pymemtrace.debug_malloc_stats` for a ``pymemtrace`` wrapper that makes this much + more useful. + + +``gc`` +""""""""""""""""""""" + +The :py:mod:`gc` controls the Python garbage collector. +Turning the garbage collector off with ``gc.disable()`` is worth trying to see what effect, if any, it has. + +``tracemalloc`` +""""""""""""""""""""" + +:py:mod:`tracemalloc` is a useful module that can trace memory blocks allocate by Python. +It is invasive and using it consumes a significant amount of memory itself. +See :ref:`memory_leaks.pymemtrace.trace_malloc` for a ``pymemtrace`` wrapper that makes this much more useful by +report changes to Python's memory allocator.. + +.. index:: single: Memory Leaks; Third Party Python Tools + +Third Party Modules +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``psutil`` +""""""""""""""""""""" + +``psutil`` is an excellent, third party, package that can report high level information on a process. +`psutil on PyPi `_ + +Here are some notes on the +`cost of computing `_ +the Resident Set Size (RSS). + + +.. _objgraph: https://pypi.org/project/objgraph/ + +``objgraph`` +""""""""""""""""""""" + +`objgraph`_ is a wrapper around the Python garbage collector that can take a snapshot of the Python objects in scope. +This is quite invasive and expensive but can be very useful in specific cases. +It does some excellent visualisations using graphviz, xdot etc. + + +.. _guppy_3: https://pypi.org/project/guppy3/ +.. _guppy_3_source: https://github.com/zhuyifei1999/guppy3/ + +``Guppy 3`` +""""""""""""""""""""" + +`guppy_3`_ (source: `guppy_3_source`_) is a project that gives a highly detailed picture of the current state of the +Python heap. + + +.. _memory_profiler: http://pypi.python.org/pypi/memory_profiler +.. _memory_profiler_src: https://github.com/pythonprofilers/memory_profiler + +``memory_profiler`` +""""""""""""""""""""" + +Sadly `memory_profiler`_ (source: `memory_profiler_src`_) this excellent project is no longer actively maintained +(volunteers welcome, contact f@bianp.net). +It gives an annotated version of your source code line by line with the memory usage. +It is pure Python and relies just on ``psutil``. + + +.. index:: single: Memory Leaks; Python Debugging Tools + +Debugging Tools +------------------ + +Debugging Python (and C/C++ extensions) with GDB: + +* GDB support for Python: ``_ +* Python debugging with GDB: ``_ + and ``_ +* Python debugging tools: ``_ + + +Building a Debug Version of Python +--------------------------------------- + +This is an essential technique however it is limited, due to speed, to a development environment rather than in +production. + +Building a debug version of Python in a variety of forms: +``_ + +Building a DTrace aware version of Python: ``_ +Here are some examples of using +`DTrace with pymemtrace `_ with some +`technical notes `_ on using DTrace. diff --git a/doc/sphinx/source/miscellaneous.rst b/doc/sphinx/source/miscellaneous.rst index db19862..fda42d1 100644 --- a/doc/sphinx/source/miscellaneous.rst +++ b/doc/sphinx/source/miscellaneous.rst @@ -4,10 +4,18 @@ .. toctree:: :maxdepth: 2 +.. _miscellaneous: + +.. index:: + single: Miscellaneous + ==================================== Miscellaneous ==================================== +This chapter covers various miscellaneous issues that the author has found with creating Python Extensions over the +years. + ------------------------------------ No ``PyInit_...`` Function Found ------------------------------------ @@ -39,3 +47,35 @@ And the binary now looks like this: $ nm -m Foo.cpython-36m-darwin.so | grep Init 00000000000010d0 (__TEXT,__text) external _PyInit_Foo + +.. _miscellaneous_migration_python_c: + +--------------------------------------- +Migrating from Python to a C Extension +--------------------------------------- + +Suppose you have followed my advice in :ref:`introduction_summary_advice` in that you write you code in Python first +then, when profiling shows the slow spots, rewrite in C. +You might not want to do this all at once so here is a technique that allows you to migrate to C in a flexible way, say +over a number of releases, without you users having to change *their* code. + +Suppose you have a bunch of functions and classes in a Python module ``spam.py``. +Then take all that Python code an put it in a file, say, ``py_spam.py``. +Now create an empty C Extension calling it, say, ``c_spam``. + +Change ``spam.py`` to be merely: + +.. code-block:: python + + from py_spam import * + from c_spam import * + +Your users, including your test code, just uses ``import spam`` and they get all of ``py_spam`` for now and nothing +from ``c_spam`` as it is empty. + +You can now, judiciously, add functionality and classes to ``c_spam`` and your users will automatically get those as +``spam`` overwrites the appropriate imports from ``py_spam`` with the ones from ``c_spam``. + + + + diff --git a/doc/sphinx/source/module_globals.rst b/doc/sphinx/source/module_globals.rst index 9e63d16..6c6d352 100644 --- a/doc/sphinx/source/module_globals.rst +++ b/doc/sphinx/source/module_globals.rst @@ -4,13 +4,18 @@ .. toctree:: :maxdepth: 2 +.. index:: + single: Module Globules; Setting + single: Module Globules; Getting + ==================================== Setting and Getting Module Globals ==================================== This section describes how you create and access module globals from Python C Extensions. -In this module, written as a Python extension in C, we are going to have a string, int, list, tuple and dict in global scope. In the C code we firstly define names for them: +In this module, written as a Python extension in C, we are going to have a string, int, list, tuple and dict in global +scope. In the C code we firstly define names for them: .. code-block:: c @@ -27,17 +32,22 @@ These are the names of the objects that will appear in the Python module:: >>> dir(cModuleGlobals) ['INT', 'LST', 'MAP', 'STR', 'TUP', '__doc__', '__file__', '__loader__', '__name__', '__package__', 'print'] + +.. index:: + single: Module Globules; Initialising + ------------------------------------ Initialising Module Globals ------------------------------------ -This is the module declaration, it will be called ``cModuleGlobals`` and has just one function; ``print()`` that will access the module globals from C: +This is the module declaration, it will be called ``cModuleGlobals`` and has just one function; ``print()`` that will +access the module globals from C: .. code-block:: c static PyMethodDef cModuleGlobals_methods[] = { - {"print", (PyCFunction)_print_globals, METH_NOARGS, - "Access and print out th globals." + {"print", (PyCFunction)print_globals, METH_NOARGS, + "Access and print out the globals." }, {NULL, NULL, 0, NULL} /* Sentinel */ }; @@ -82,8 +92,8 @@ The module initialisation code is next, this uses the Python C API to create the if (PyModule_AddObject(m, NAME_LST, Py_BuildValue("[iii]", 66, 68, 73))) { goto except; } - /* An invented convenience function for this dict. */ - if (_add_map_to_module(m)) { + /* An invented convenience function for this dict. See below. */ + if (add_map_to_module(m)) { goto except; } goto finally; @@ -101,21 +111,31 @@ The dict is added in a separate C function merely for readability: /* Add a dict of {str : int, ...}. * Returns 0 on success, 1 on failure. */ - int _add_map_to_module(PyObject *module) { + int add_map_to_module(PyObject *module) { int ret = 0; PyObject *pMap = NULL; - + PyObject *key = NULL; + PyObject *val = NULL; + pMap = PyDict_New(); - if (! pMap) { + if (!pMap) { goto except; } /* Load map. */ - if (PyDict_SetItem(pMap, PyBytes_FromString("66"), PyLong_FromLong(66))) { + key = PyBytes_FromString("66"); + val = PyLong_FromLong(66); + if (PyDict_SetItem(pMap, key, val)) { goto except; } - if (PyDict_SetItem(pMap, PyBytes_FromString("123"), PyLong_FromLong(123))) { + Py_XDECREF(key); + Py_XDECREF(val); + key = PyBytes_FromString("123"); + val = PyLong_FromLong(123); + if (PyDict_SetItem(pMap, key, value)) { goto except; } + Py_XDECREF(key); + Py_XDECREF(val); /* Add map to module. */ if (PyModule_AddObject(module, NAME_MAP, pMap)) { goto except; @@ -124,15 +144,20 @@ The dict is added in a separate C function merely for readability: goto finally; except: Py_XDECREF(pMap); + Py_XDECREF(key); + Py_XDECREF(val); ret = 1; finally: return ret; } ------------------------------------ -Getting and Setting Module Globals +Getting and Setting ------------------------------------ +.. index:: + single: Module Globules; Getting From Python + ^^^^^^^^^^^^^^^^^^ From Python ^^^^^^^^^^^^^^^^^^ @@ -153,15 +178,20 @@ Once the module is built we can access the globals from Python as usual:: >>> cModuleGlobals.MAP {b'123': 123, b'asd': 9, b'66': 66} +.. index:: + single: Module Globules; Getting From C + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Getting Module Globals From C ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Accessing Python module globals from C is a little bit more tedious as we are getting borrowed references from the modules ``__dict__`` and we should increment and decrement them appropriately. Here we print out the global ``INT`` as both a Python object and a 'C' ``long``: +Accessing Python module globals from C is a little bit more tedious as we are getting borrowed references from the +modules ``__dict__`` and we should increment and decrement them appropriately. +Here we print out the global ``INT`` as both a Python object and a 'C' ``long``: .. code-block:: c - static PyObject *_print_global_INT(PyObject *pMod) { + static PyObject *print_global_INT(PyObject *pMod) { PyObject *ret = NULL; PyObject *pItem = NULL; long val; @@ -202,7 +232,7 @@ Accessing Python module globals from C is a little bit more tedious as we are ge return ret; } -From Python we would see this (C's ``_print_global_INT()`` is mapped to Python's ``cModuleGlobals.printINT()``): +From Python we would see this (C's ``print_global_INT()`` is mapped to Python's ``cModuleGlobals.printINT()``): >>> import cModuleGlobals >>> cModuleGlobals.printINT() @@ -210,19 +240,30 @@ From Python we would see this (C's ``_print_global_INT()`` is mapped to Python's Integer: "INT" 42 C long: 42 +.. index:: + single: Module Globules; Setting From C + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Setting Module Globals From C ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This is similar to the get code above but using ``int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val)`` where val will be a *stolen* reference: +This is similar to the get code above but using ``int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val)`` +where val will **not** be a *stolen* reference thus must be created as a temporary +and subsequently decref'd: .. code-block:: c static PyObject *some_set_function(PyObject *pMod) { PyObject *ret = NULL; long val = ...; /* Some computed value. */ - - if (PyDict_SetItemString(PyModule_GetDict(pMod), NAME_INT, PyLong_FromLong(val))) { + + PyObject *py_long = PyLong_FromLong(val); + if (! py_long) { + goto except; + } + /* PyDict_SetItemString does not steal a reference to py_long + so we have to decref the temporary. */ + if (PyDict_SetItemString(PyModule_GetDict(pMod), NAME_INT, py_long)) { PyErr_Format(PyExc_AttributeError, "Can not set Module '%s' attibute '%s'.", \ PyModule_GetName(pMod), NAME_INT @@ -239,7 +280,7 @@ This is similar to the get code above but using ``int PyDict_SetItemString(PyObj Py_XDECREF(ret); ret = NULL; finally: + /* See comment above about PyDict_SetItemString(). */ + Py_XDECREF(py_long); return ret; } - - diff --git a/doc/sphinx/source/new_types.rst b/doc/sphinx/source/new_types.rst index 47f374f..7775048 100644 --- a/doc/sphinx/source/new_types.rst +++ b/doc/sphinx/source/new_types.rst @@ -4,25 +4,40 @@ .. toctree:: :maxdepth: 2 -==================================== +.. _chapter_creating_new_types: + +.. index:: + single: New Types; Creating + +************************************ Creating New Types -==================================== +************************************ -The creation of new extension types (AKA 'classes') is pretty well described in the Python documentation `tutorial `_ and -`reference `_. This section just describes a rag bag of tricks and examples. +The creation of new extension types (AKA 'classes') is pretty well described in the Python documentation +`tutorial `_ and +`reference `_. +This section is a cookbook of tricks and examples. ------------------------------------- +==================================== Properties ------------------------------------- +==================================== -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. index:: + single: New Types; Existing Python Properties + single: New Types; Existing C Properties + +------------------------------------ Referencing Existing Properties -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------ If the property is part of the extension type then it is fairly easy to make it directly accessible as -`described here `_ +`described here `_ + +.. note:: Terminology + + In this section "property", "attribute" and "field" are used interchangeably. -For example the ``Noddy`` struct has a Python object (a string) and a C object (an int): +For example the ``Noddy`` struct has a Python object (a Python string) and a C object (an C int): .. code-block:: c @@ -58,9 +73,13 @@ And the type struct must reference this array of ``PyMemberDef`` thus: `Reference to PyMemberdef. `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. index:: + single: New Types; Dynamic Python Properties + single: New Types; Created Python Properties + +-------------------------- Created Properties -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- If the properties are not directly accessible, for example they might need to be created, then an array of ``PyGetSetDef`` structures is used in the `PyTypeObject.tp_getset `_ slot. @@ -95,3 +114,1313 @@ And the type struct must reference this array of ``PyMemberDef`` thus: `Reference to PyGetSetDef. `_ + +==================================== +Subclassing +==================================== + +This large subject gets it own chapter: :ref:`chapter_subclassing_and_using_super`. + + +.. index:: + single: New Types; Examples + +==================================== +Examples +==================================== + +See ``src/cpy/cObject.c`` for some examples, the tests for these are in ``tests/unit/test_c_object.py``: + +- ``Null`` is a basic class that does nothing. +- ``Str`` is a subclass of the builtin ``str`` class. + + +.. index:: + single: New Types; Set Attributes Dynamically + single: New Types; Get Attributes Dynamically + single: New Types; Delete Attributes Dynamically + +---------------------------------------------------- +Setting, Getting and Deleting Attributes Dynamically +---------------------------------------------------- + +In ``src/cpy/cObject.c`` there is an example, ``ObjectWithAttributes``, which is a class that can set, get, delete +attributes dynamically. + +Firstly the object declaration: + +.. code-block:: c + + typedef struct { + PyObject_HEAD + /* Attributes dictionary, NULL on construction, + * will be populated by MyObj_getattro. */ + PyObject *x_attr; + } ObjectWithAttributes; + +Then some type checking that requires a forward declaration: + +.. code-block:: c + + /** Forward declaration. */ + static PyTypeObject ObjectWithAttributes_Type; + #define ObjectWithAttributes_Check(v) (Py_TYPE(v) == &ObjectWithAttributes_Type) + +The ``__new__`` and ``__del__`` methods: + +.. code-block:: c + + static ObjectWithAttributes * + ObjectWithAttributes_new(PyObject *Py_UNUSED(arg)) { + ObjectWithAttributes *self; + self = PyObject_New(ObjectWithAttributes, &ObjectWithAttributes_Type); + if (self == NULL) { + return NULL; + } + self->x_attr = NULL; + return self; + } + + /* ObjectWithAttributes methods */ + static void + ObjectWithAttributes_dealloc(ObjectWithAttributes *self) { + Py_XDECREF(self->x_attr); + PyObject_Del(self); + } + +Add an empty method for demonstration: + +.. code-block:: c + + static PyObject * + ObjectWithAttributes_demo(ObjectWithAttributes *Py_UNUSED(self), PyObject *args) { + if (!PyArg_ParseTuple(args, ":demo")) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; + } + + static PyMethodDef ObjectWithAttributes_methods[] = { + {"demo", (PyCFunction) ObjectWithAttributes_demo, METH_VARARGS, + PyDoc_STR("demo() -> None")}, + {NULL, NULL, 0, NULL} /* sentinel */ + }; + +Now the methods to get and set attribute: + +.. code-block:: c + + static PyObject * + ObjectWithAttributes_getattro(ObjectWithAttributes *self, PyObject *name) { + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItem(self->x_attr, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + } + return PyObject_GenericGetAttr((PyObject *) self, name); + } + + static int + ObjectWithAttributes_setattr(ObjectWithAttributes *self, char *name, PyObject *v) { + if (self->x_attr == NULL) { + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) + return -1; + } + if (v == NULL) { + int rv = PyDict_DelItemString(self->x_attr, name); + if (rv < 0) + PyErr_SetString(PyExc_AttributeError, + "delete non-existing ObjectWithAttributes attribute"); + return rv; + } else + /* v is a borrowed reference, + * then PyDict_SetItemString() does NOT steal it + * so nothing to do. */ + return PyDict_SetItemString(self->x_attr, name, v); + } + +Finally the type declaration. Note this is the complete type declaration with compiler declarations to suit +Python versions 3.6 to 3.13: + +.. code-block:: c + + static PyTypeObject ObjectWithAttributes_Type = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "cObject.ObjectWithAttributes", /*tp_name*/ + sizeof(ObjectWithAttributes), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor) ObjectWithAttributes_dealloc, /*tp_dealloc*/ + #if PY_MINOR_VERSION < 8 + 0, /*tp_print*/ + #else + 0, /* Py_ssize_t tp_vectorcall_offset; */ + #endif + (getattrfunc) 0, /*tp_getattr*/ + (setattrfunc) ObjectWithAttributes_setattr, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + (getattrofunc) ObjectWithAttributes_getattro, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + ObjectWithAttributes_methods, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + // PyType_GenericNew, /*tp_new*/ + (newfunc) ObjectWithAttributes_new, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + NULL, /* tp_bases */ + NULL, /* tp_mro */ + NULL, /* tp_cache */ + NULL, /* tp_subclasses */ + NULL, /* tp_weaklist */ + NULL, /* tp_del */ + 0, /* tp_version_tag */ + NULL, /* tp_finalize */ + #if PY_MINOR_VERSION > 7 + NULL, /* tp_vectorcall */ + #endif + #if PY_MINOR_VERSION == 8 + 0, /*tp_print*/ + #endif + #if PY_MINOR_VERSION >= 12 + '\0', /* unsigned char tp_watched */ + #if PY_MINOR_VERSION >= 13 + 0, /* uint16_t tp_versions_used */ + #endif + #endif + }; + +Finally add this to the module (partial code): + +.. code-block:: c + + static struct PyModuleDef cObject = { + PyModuleDef_HEAD_INIT, + "cObject", + module_doc, + -1, + cObject_functions, + NULL, + NULL, + NULL, + NULL + }; + + PyMODINIT_FUNC + PyInit_cObject(void) { + PyObject *m = NULL; + + /* Create the module and add the functions */ + m = PyModule_Create(&cObject); + if (m == NULL) { + goto fail; + } + /* Add some symbolic constants to the module */ + if (ErrorObject == NULL) { + ErrorObject = PyErr_NewException("cObject.error", NULL, NULL); + if (ErrorObject == NULL) + goto fail; + } + Py_INCREF(ErrorObject); + if (PyModule_AddObject(m, "error", ErrorObject)) { + goto fail; + } + /* Finalize the type object including setting type of the new type + * object; doing it here is required for portability, too. */ + if (PyType_Ready(&ObjectWithAttributes_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "ObjectWithAttributes", + (PyObject *) &ObjectWithAttributes_Type)) { + goto fail; + } + /* More here ... */ + return m; + fail: + Py_XDECREF(m); + return NULL; + } + +This can be tested thus, in ``tests/unit/test_c_object.py``: + +.. code-block:: python + + import pytest + + from cPyExtPatt import cObject + + def test_ObjectWithAttributes_set_and_get(): + obj = cObject.ObjectWithAttributes() + obj.some_attr = 'Some attribute' + assert hasattr(obj, 'some_attr') + assert obj.some_attr == 'Some attribute' + + + def test_ObjectWithAttributes_set_and_del(): + obj = cObject.ObjectWithAttributes() + obj.some_attr = 'Some attribute' + assert hasattr(obj, 'some_attr') + delattr(obj, 'some_attr') + assert not hasattr(obj, 'some_attr') + with pytest.raises(AttributeError) as err: + obj.some_attr + assert err.value.args[0] == "'cObject.ObjectWithAttributes' object has no attribute 'some_attr'" + + +.. _PySequence_Check(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_Check +.. _PySequence_GetItem(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_GetItem +.. _PySequence_SetItem(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_SetItem +.. _PySequence_Contains(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_Contains +.. _PySequence_Concat(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_Concat +.. _PySequence_InPlaceConcat(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_InPlaceConcat +.. _PySequence_Repeat(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_Repeat +.. _PySequence_InPlaceRepeat(): https://docs.python.org/3/c-api/sequence.html#c.PySequence_InPlaceRepeat + +.. _PyObject_SetItem(): https://docs.python.org/3/c-api/object.html#c.PyObject_SetItem +.. _PyObject_DelItem(): https://docs.python.org/3/c-api/object.html#c.PyObject_DelItem + +.. _PySequenceMethods: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods + +.. _sq_length: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_length +.. _sq_concat: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_concat +.. _sq_repeat: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_repeat +.. _sq_item: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_item +.. _sq_ass_item: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_ass_item +.. _sq_ass_contains: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_ass_contains +.. _sq_inplace_concat: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_inplace_concat +.. _sq_inplace_repeat: https://docs.python.org/3/c-api/typeobj.html#c.PySequenceMethods.sq_inplace_repeat + +.. _lenfunc: https://docs.python.org/3/c-api/typeobj.html#c.lenfunc +.. _binaryfunc: https://docs.python.org/3/c-api/typeobj.html#c.binaryfunc +.. _ssizeargfunc: https://docs.python.org/3/c-api/typeobj.html#c.ssizeargfunc +.. _ssizeobjargproc: https://docs.python.org/3/c-api/typeobj.html#c.ssizeobjargproc +.. _objobjproc: https://docs.python.org/3/c-api/typeobj.html#c.objobjproc + + +.. index:: + single: Sequence Types + single: New Types; Sequence Types + +========================= +Emulating Sequence Types +========================= + +This section describes how to make an object act like a sequence using +`tp_as_sequence `_. +See also `Sequence Object Structures `_. +This allows your objects to have set and get methods like a list. + +As an example here is an extension that can represent a sequence of longs in C with a CPython sequence interface. +The code is in ``src/cpy/Object/cSeqObject.c``. + +First the class declaration: + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + #include + #include "structmember.h" + + typedef struct { + PyObject_HEAD + long *array_long; + ssize_t size; + } SequenceLongObject; + + +Then the equivalent of ``__new__``: + +.. code-block:: c + + static PyObject * + SequenceLongObject_new(PyTypeObject *type, PyObject *Py_UNUSED(args), + PyObject *Py_UNUSED(kwds)) { + SequenceLongObject *self; + self = (SequenceLongObject *) type->tp_alloc(type, 0); + if (self != NULL) { + assert(!PyErr_Occurred()); + self->size = 0; + self->array_long = NULL; + } + return (PyObject *) self; + } + +And the equivalent of ``__init__`` that takes a Python sequence of ints: + +.. code-block:: c + + static int + SequenceLongObject_init(SequenceLongObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return -1; + } + if (!PySequence_Check(sequence)) { + return -2; + } + self->size = PySequence_Length(sequence); + self->array_long = malloc(self->size * sizeof(long)); + if (!self->array_long) { + return -3; + } + for (Py_ssize_t i = 0; i < self->size; ++i) { + // New reference. + PyObject *py_value = PySequence_GetItem(sequence, i); + if (PyLong_Check(py_value)) { + self->array_long[i] = PyLong_AsLong(py_value); + Py_DECREF(py_value); + } else { + PyErr_Format( + PyExc_TypeError, + "Argument [%zd] must be a int, not type %s", + i, + Py_TYPE(sequence)->tp_name + ); + // Clean up on error. + free(self->array_long); + self->array_long = NULL; + Py_DECREF(py_value); + return -4; + } + } + return 0; + } + +And de-allocation: + +.. code-block:: c + + static void + SequenceLongObject_dealloc(SequenceLongObject *self) { + free(self->array_long); + Py_TYPE(self)->tp_free((PyObject *) self); + } + +.. index:: + single: Sequence Types; Function Table + +--------------------------- +The Sequence Function Table +--------------------------- + +The sequence functions are a table of function pointers in the `PySequenceMethods`_ table. +Here is a summary of the fields, each is described in more detail below: + +.. list-table:: Sequence Methods + :widths: 20 80 + :header-rows: 1 + + * - Member + - Description + * - `sq_length`_ + - Returns the length of the sequence. + * - `sq_concat`_ + - Takes two sequences and returns a new third one with the first and second concatenated. + * - `sq_repeat`_ + - Returns a new sequence with the old one repeated n times. + * - `sq_item`_ + - Returns a *new* reference to the n'th item in the sequence. + Negative indexes are handled appropriately. + Used by `PySequence_GetItem()`_. + This is a fairly crucial implementation for a sequence as `PySequence_Check()`_ detects this to decide if the + object is a sequence. + * - `sq_ass_item`_ + - Sets the the n'th item in the sequence. + If the value is NULL the item is deleted and the sequence concatenated (thus called by `PyObject_DelItem()`_). + Negative indexes are handled appropriately. + Used by `PyObject_SetItem()`_. + * - `sq_ass_contains`_ + - Returns non-zero if the sequence contains the given object. + Used by `PySequence_Contains()`_. + This slot may be left to NULL, in this case PySequence_Contains() simply traverses the sequence until it + finds a match. + * - `sq_inplace_concat`_ + - Provides in-place concatenation, for example ``+=``. + If this slot is ``NULL`` then `PySequence_Concat()`_ will be used returning a new object. + * - `sq_inplace_repeat`_ + - Provides in-place concatenation, for example ``+=``. + Returns the existing sequence repeated n times. + If this slot is ``NULL`` then `PySequence_Repeat()`_ will be used returning a new object. + +Here are the individual functions, their implementation and the test code. + + +.. index:: + single: sq_length + single: New Types; sq_length + single: Sequence Types; sq_length + +--------------- +``sq_length`` +--------------- + +.. list-table:: Sequence Methods: ``sq_length`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_length`_ + * - Function type + - `lenfunc`_ + * - Function signature + - ``Py_ssize_t (*lenfunc)(PyObject*)`` + * - Description + - Returns the length of the sequence. + +Implementation +-------------- + +In ``src/cpy/Object/cSeqObject.c``: + +.. code-block:: c + + static Py_ssize_t + SequenceLongObject_sq_length(PyObject *self) { + return ((SequenceLongObject *) self)->size; + } + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + def test_SequenceLongObject_len(): + obj = cSeqObject.SequenceLongObject([7, 4, 1, ]) + assert len(obj) == 3 + +.. index:: + single: sq_concat + single: New Types; sq_concat + single: Sequence Types; sq_concat + +--------------- +``sq_concat`` +--------------- + +.. list-table:: Sequence Methods: ``sq_concat`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_concat`_ + * - Function type + - `binaryfunc`_ + * - Function signature + - ``PyObject *(*binaryfunc)(PyObject*, PyObject*)`` + * - Description + - Takes two sequences and returns a new third one with the first and second concatenated. + This is used by the ``+`` Python operator and the `PySequence_Concat()`_ C API. + +Implementation +-------------- + +Note the use of forward references for the type checker as we need to check of the second argument is of the +correct type. +In ``src/cpy/Object/cSeqObject.c``: + +.. code-block:: c + + // Forward references + static PyTypeObject SequenceLongObjectType; + static int is_sequence_of_long_type(PyObject *op); + + /** Returns a new SequenceLongObject composed of self + other. */ + static PyObject * + SequenceLongObject_sq_concat(PyObject *self, PyObject *other) { + if (!is_sequence_of_long_type(other)) { + PyErr_Format( + PyExc_TypeError, + "%s(): argument 1 must have type \"SequenceLongObject\" not %s", + Py_TYPE(other)->tp_name + ); + return NULL; + } + PyObject *ret = SequenceLongObject_new(&SequenceLongObjectType, NULL, NULL); + if (!ret) { + assert(PyErr_Occurred()); + return NULL; + } + /* For convenience. */ + SequenceLongObject *ret_as_slo = (SequenceLongObject *) ret; + ret_as_slo->size = ((SequenceLongObject *) self)->size \ + + ((SequenceLongObject *) other)->size; + ret_as_slo->array_long = malloc(ret_as_slo->size * sizeof(long)); + if (!ret_as_slo->array_long) { + PyErr_Format( + PyExc_MemoryError, "%s(): Can not create new object.", __FUNCTION__ + ); + Py_DECREF(ret); + return NULL; + } + + ssize_t i = 0; + ssize_t ub = ((SequenceLongObject *) self)->size; + while (i < ub) { + ret_as_slo->array_long[i] = ((SequenceLongObject *) self)->array_long[i]; + i++; + } + ssize_t j = 0; + ub = ((SequenceLongObject *) other)->size; + while (j < ub) { + ret_as_slo->array_long[i] = ((SequenceLongObject *) other)->array_long[j]; + i++; + j++; + } + return ret; + } + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + def test_SequenceLongObject_concat(): + obj_a = cSeqObject.SequenceLongObject([7, 4, 1, ]) + obj_b = cSeqObject.SequenceLongObject([70, 40, 100, ]) + assert id(obj_a) != id(obj_b) + obj = obj_a + obj_b + assert id(obj) != id(obj_a) + assert id(obj) != id(obj_b) + assert len(obj) == 6 + assert list(obj) == [7, 4, 1, ] + [70, 40, 100, ] + +.. index:: + single: sq_repeat + single: New Types; sq_repeat + single: Sequence Types; sq_repeat + +--------------- +``sq_repeat`` +--------------- + +.. list-table:: Sequence Methods: ``sq_repeat`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_repeat`_ + * - Function type + - `ssizeargfunc`_ + * - Function signature + - ``PyObject *(*ssizeargfunc)(PyObject*, Py_ssize_t)`` + * - Description + - Returns a new sequence with the old one repeated the given number of times times. + This is used by the ``*`` Python operator and the `PySequence_Repeat()`_ C API. + +Implementation +-------------- + +The implementation is fairly straightforward in ``src/cpy/Object/cSeqObject.c``. +Note that ``count`` can be zero or negative: + +.. code-block:: c + + /** Return a new sequence which contains the old one repeated count times. */ + static PyObject * + SequenceLongObject_sq_repeat(PyObject *self, Py_ssize_t count) { + PyObject *ret = SequenceLongObject_new(&SequenceLongObjectType, NULL, NULL); + if (!ret) { + assert(PyErr_Occurred()); + return NULL; + } + assert(ret != self); + if (((SequenceLongObject *) self)->size > 0 && count > 0) { + /* For convenience. */ + SequenceLongObject *self_as_slo = (SequenceLongObject *) self; + SequenceLongObject *ret_as_slo = (SequenceLongObject *) ret; + ret_as_slo->size = self_as_slo->size * count; + assert(ret_as_slo->size > 0); + ret_as_slo->array_long = malloc(ret_as_slo->size * sizeof(long)); + if (!ret_as_slo->array_long) { + PyErr_Format( + PyExc_MemoryError, "%s(): Can not create new object.", __FUNCTION__ + ); + Py_DECREF(ret); + return NULL; + } + Py_ssize_t ret_index = 0; + for (Py_ssize_t i = 0; i < count; ++i) { + for (Py_ssize_t j = 0; j < self_as_slo->size; ++j) { + ret_as_slo->array_long[ret_index] = self_as_slo->array_long[j]; + ++ret_index; + } + } + } else { + /* Empty sequence. */ + } + return ret; + } + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + @pytest.mark.parametrize( + 'initial_sequence, count, expected', + ( + ([], 1, [],), + ([7, 4, 1, ], 0, [],), + ([7, 4, 1, ], -1, [],), + ([7, 4, 1, ], 1, [7, 4, 1, ],), + ([7, 4, 1, ], 2, [7, 4, 1, 7, 4, 1, ],), + ([7, 4, 1, ], 3, [7, 4, 1, 7, 4, 1, 7, 4, 1, ],), + ) + ) + def test_SequenceLongObject_repeat(initial_sequence, count, expected): + obj_a = cSeqObject.SequenceLongObject(initial_sequence) + obj = obj_a * count + print() + assert id(obj_a) != id(obj) + assert list(obj) == expected + assert list(obj) == (list(obj_a) * count) + + +.. index:: + single: sq_item + single: New Types; sq_item + single: Sequence Types; sq_item + +--------------- +``sq_item`` +--------------- + +`sq_item`_ gives read access to an indexed member. + +.. list-table:: Sequence Methods: ``sq_item`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_item`_ + * - Function type + - `ssizeargfunc`_ + * - Function signature + - ``PyObject *(*ssizeargfunc)(PyObject*, Py_ssize_t)`` + * - Description + - Returns a *new* reference to the n'th item in the sequence. + Negative indexes are handled appropriately. + Used by `PySequence_GetItem()`_. + This is a fairly crucial implementation for a sequence as `PySequence_Check()`_ detects this to decide if the + object is a sequence. + +Implementation +-------------- + +In ``src/cpy/Object/cSeqObject.c``: + +.. code-block:: c + + /** Returns a new reference to an indexed item in a sequence. */ + static PyObject * + SequenceLongObject_sq_item(PyObject *self, Py_ssize_t index) { + Py_ssize_t my_index = index; + if (my_index < 0) { + my_index += SequenceLongObject_sq_length(self); + } + // Corner case example: len(self) == 0 and index < 0 + if (my_index < 0 || my_index >= SequenceLongObject_sq_length(self)) { + PyErr_Format( + PyExc_IndexError, + "Index %ld is out of range for length %ld", + index, + SequenceLongObject_sq_length(self) + ); + return NULL; + } + return PyLong_FromLong(((SequenceLongObject *) self)->array_long[my_index]); + } + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ([7, 4, 1, ], 0, 7,), + ([7, 4, 1, ], 1, 4,), + ([7, 4, 1, ], 2, 1,), + ([7, 4, 1, ], -1, 1,), + ([7, 4, 1, ], -2, 4,), + ([7, 4, 1, ], -3, 7,), + ) + ) + def test_SequenceLongObject_item(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + assert obj[index] == expected + +And failure modes: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ([], 0, 'Index 0 is out of range for length 0',), + ([], -1, 'Index -1 is out of range for length 0',), + ([1, ], 2, 'Index 2 is out of range for length 1',), + ) + ) + def test_SequenceLongObject_item_raises(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + with pytest.raises(IndexError) as err: + obj[index] + assert err.value.args[0] == expected + + +.. index:: + single: sq_ass_item + single: New Types; sq_ass_item + single: Sequence Types; sq_ass_item + +--------------- +``sq_ass_item`` +--------------- + +`sq_ass_item`_ gives write and delete access to an indexed member. + +.. list-table:: Sequence Methods: ``sq_ass_item`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_ass_item`_ + * - Function type + - `ssizeobjargproc`_ + * - Function signature + - ``int (*ssizeobjargproc)(PyObject*, Py_ssize_t, PyObject*)`` + * - Description + - Sets the the n'th item in the sequence. + Returns 0 on success, -1 on failure. + If the value is NULL the item is deleted and the sequence concatenated (thus called by `PyObject_DelItem()`_). + Negative indexes are handled appropriately. + Used by `PyObject_SetItem()`_. + +.. index:: + single: Documentation Lacunae; sq_ass_item + +Implementation +-------------- + +.. warning:: + + There is an undocumented feature when using `sq_ass_item`_ from `PyObject_SetItem()`_ and `PyObject_DelItem()`_ + when using negative indexes and when the index is *out of range*. + + In that case, before the `sq_ass_item`_ function is called the index will have had the sequence length added to it. + + For example if the sequence length is 3 and the given index is -4 then the index that the `sq_ass_item` function + receives is -1. + If the given index is -5 then the index that the `sq_ass_item`_ function receives is -2. + + Thus the slightly odd code below to fix this problem. + Failing to do this will mean out of range errors will not be detected by the `sq_ass_item`_ function and any + error message will be wrong. + + +In ``src/cpy/Object/cSeqObject.c``: + +.. code-block:: c + + static int + SequenceLongObject_sq_ass_item(PyObject *self, Py_ssize_t index, PyObject *value) { + /* See warning above. */ + if (index < 0) { + index -= SequenceLongObject_sq_length(self); + } + + Py_ssize_t my_index = index; + if (my_index < 0) { + my_index += SequenceLongObject_sq_length(self); + } + // Corner case example: len(self) == 0 and index < 0 + if (my_index < 0 || my_index >= SequenceLongObject_sq_length(self)) { + PyErr_Format( + PyExc_IndexError, + "Index %ld is out of range for length %ld", + index, + SequenceLongObject_sq_length(self) + ); + return -1; + } + if (value != NULL) { + /* Just set the value. */ + if (!PyLong_Check(value)) { + PyErr_Format( + PyExc_TypeError, + "sq_ass_item value needs to be an int, not type %s", + Py_TYPE(value)->tp_name + ); + return -1; + } + ((SequenceLongObject *) self)->array_long[my_index] = PyLong_AsLong(value); + } else { + /* Delete the value. */ + /* For convenience. */ + SequenceLongObject *self_as_slo = (SequenceLongObject *) self; + /* Special case: deleting the only item in the array. */ + if (self_as_slo->size == 1) { + free(self_as_slo->array_long); + self_as_slo->array_long = NULL; + self_as_slo->size = 0; + } else { + /* Delete the value and re-compose the array. */ + long *new_array = malloc((self_as_slo->size - 1) * sizeof(long)); + if (!new_array) { + PyErr_Format( + PyExc_MemoryError, + "sq_ass_item can not allocate new array. %s#%d", + __FILE__, __LINE__ + ); + return -1; + } + /* Copy up to the index. */ + Py_ssize_t index_new_array = 0; + for (Py_ssize_t i = 0; i < my_index; ++i, ++index_new_array) { + new_array[index_new_array] = self_as_slo->array_long[i]; + } + /* Copy past the index. */ + for ( + Py_ssize_t i = my_index + 1; + i < self_as_slo->size; + ++i, ++index_new_array) { + new_array[index_new_array] = self_as_slo->array_long[i]; + } + + free(self_as_slo->array_long); + self_as_slo->array_long = new_array; + --self_as_slo->size; + } + } + return 0; + } + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py`` which includes failure modes. +First setting a value: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, index, value, expected', + ( + ([7, 4, 1, ], 0, 14, [14, 4, 1, ],), + ([7, 4, 1, ], -1, 14, [7, 4, 14, ],), + ([7, 4, 1, ], -2, 14, [7, 14, 1, ],), + ([7, 4, 1, ], -3, 14, [14, 4, 1, ],), + ) + ) + def test_SequenceLongObject_setitem(initial_sequence, index, value, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + obj[index] = value + assert list(obj) == expected + + +Setting a value with an out of range index: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ([7, 4, 1, ], 3, 'Index 3 is out of range for length 3',), + ([7, 4, 1, ], -4, 'Index -4 is out of range for length 3',), + ) + ) + def test_SequenceLongObject_setitem_raises(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + with pytest.raises(IndexError) as err: + obj[index] = 100 + assert err.value.args[0] == expected + + +Deleting a value: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ([7, ], 0, [],), + ([7, ], -1, [],), + ([7, 4, 1, ], 1, [7, 1, ],), + ([7, 4, ], 0, [4, ],), + ([7, 4, 1, ], -1, [7, 4, ],), + ([7, 4, 1, ], -2, [7, 1, ],), + ([7, 4, 1, ], -3, [4, 1, ],), + ) + ) + def test_SequenceLongObject_delitem(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + del obj[index] + assert list(obj) == expected + + +Deleting a value with an out of range index: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ([], 0, 'Index 0 is out of range for length 0',), + ([], -1, 'Index -1 is out of range for length 0',), + ([7, ], 1, 'Index 1 is out of range for length 1',), + ([7, ], -3, 'Index -3 is out of range for length 1',), + ) + ) + def test_SequenceLongObject_delitem_raises(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + with pytest.raises(IndexError) as err: + del obj[index] + assert err.value.args[0] == expected + + +.. index:: + single: sq_ass_contains + single: New Types; sq_ass_contains + single: Sequence Types; sq_ass_contains + +------------------- +``sq_ass_contains`` +------------------- + +.. list-table:: Sequence Methods: ``sq_ass_contains`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_ass_contains`_ + * - Function type + - `objobjproc`_ + * - Function signature + - ``int (*objobjproc)(PyObject*, PyObject*)`` + * - Description + - Returns non-zero if the sequence contains the given object. + If an item in o is equal to value, return 1, otherwise return 0. On error, return -1. + This is equivalent to the Python expression ``value in o``. + Used by `PySequence_Contains()`_. + This slot may be left to NULL, in this case `PySequence_Contains()`_ simply traverses the sequence until it + finds a match. + +Implementation +-------------- + +In ``src/cpy/Object/cSeqObject.c``: + +.. code-block:: c + + /** If an item in self is equal to value, return 1, otherwise return 0. + * On error, return -1. */ + static int + SequenceLongObject_sq_contains(PyObject *self, PyObject *value) { + fprintf( + stdout, "%s()#%d: self=%p value=%p\n", + __FUNCTION__, __LINE__, (void *) self, (void *) value + ); + if (!PyLong_Check(value)) { + /* Alternates: Could raise TypeError or return -1. + * Here we act benignly! */ + return 0; + } + long c_value = PyLong_AsLong(value); + /* For convenience. */ + SequenceLongObject *self_as_slo = (SequenceLongObject *) self; + for (Py_ssize_t i = 0; i < SequenceLongObject_sq_length(self); ++i) { + if (self_as_slo->array_long[i] == c_value) { + return 1; + } + } + return 0; + } + +.. note:: + + Whilst ``SequenceLongObject_sq_contains()`` returns 0 or 1 however Python code such as ``value in obj`` converts + this to ``False`` and ``True`` respectively + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, value, expected', + ( + ([7, ], 0, False,), + ([7, ], 7, True,), + ([1, 4, 7, ], 7, True,), + ) + ) + def test_SequenceLongObject_contains(initial_sequence, value, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + result = value in obj + assert result == expected + +.. index:: + single: sq_inplace_concat + single: New Types; sq_inplace_concat + single: Sequence Types; sq_inplace_concat + +--------------------- +``sq_inplace_concat`` +--------------------- + +.. list-table:: Sequence Methods: ``sq_inplace_concat`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_inplace_concat`_ + * - Function type + - `binaryfunc`_ + * - Function signature + - ``PyObject *(*binaryfunc)(PyObject*, PyObject*)`` + * - Description + - Provides in-place concatenation, for example ``+=``. + If this slot is ``NULL`` then `PySequence_InPlaceConcat()`_ will use `PySequence_Concat()`_. + +Implementation +-------------- + +If `sq_concat`_ is implemented then this need not be implemented +In ``src/cpy/Object/cSeqObject.c`` this is not implemented to show this mechanism at work. + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + def test_SequenceLongObject_concat_inplace(): + obj_a = cSeqObject.SequenceLongObject([7, 4, 1, ]) + obj_b = cSeqObject.SequenceLongObject([70, 40, 100, ]) + assert id(obj_a) != id(obj_b) + obj_a += obj_b + assert len(obj_a) == 6 + assert list(obj_a) == [7, 4, 1, ] + [70, 40, 100, ] + + +.. index:: + single: sq_inplace_repeat + single: New Types; sq_inplace_repeat + single: Sequence Types; sq_inplace_repeat + +--------------------- +``sq_inplace_repeat`` +--------------------- + +.. list-table:: Sequence Methods: ``sq_inplace_repeat`` + :widths: 20 80 + :header-rows: 0 + + * - Member + - `sq_inplace_repeat`_ + * - Function type + - `ssizeargfunc`_ + * - Function signature + - ``PyObject *(*ssizeargfunc)(PyObject*, Py_ssize_t)`` + * - Description + - Provides in-place concatenation, for example ``+=``. + Returns the existing sequence repeated n times. + If this slot is ``NULL`` then `PySequence_Repeat()`_ will be used returning a new object. + +Implementation +-------------- + +If `sq_repeat`_ is implemented then this need not be implemented +In ``src/cpy/Object/cSeqObject.c`` this is not implemented to show this mechanism at work. + +Tests +-------------- + +Tests are in ``tests/unit/test_c_seqobject.py``: + +.. code-block:: python + + from cPyExtPatt import cSeqObject + + @pytest.mark.parametrize( + 'initial_sequence, count, expected', + ( + ([], 1, [],), + ([7, 4, 1, ], 0, [],), + ([7, 4, 1, ], -1, [],), + ([7, 4, 1, ], 1, [7, 4, 1, ],), + ([7, 4, 1, ], 2, [7, 4, 1, 7, 4, 1, ],), + ([7, 4, 1, ], 3, [7, 4, 1, 7, 4, 1, 7, 4, 1, ],), + ) + ) + def test_SequenceLongObject_repeat_inplace(initial_sequence, count, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + obj *= count + assert list(obj) == expected + assert list(obj) == (initial_sequence * count) + +.. index:: + single: PySequenceMethods + single: New Types; PySequenceMethods + single: Sequence Types; Function Table + +--------------------- +The Function Table +--------------------- + +All these functions are gathered together in a `PySequenceMethods`_ table: + +.. code-block:: c + + static PySequenceMethods SequenceLongObject_sequence_methods = { + .sq_length = (lenfunc)SequenceLongObject_sq_length, + .sq_concat = (binaryfunc)SequenceLongObject_sq_concat, + .sq_repeat = (ssizeargfunc)SequenceLongObject_sq_repeat, + .sq_item = (ssizeargfunc)SequenceLongObject_sq_item, + .sq_ass_item = (ssizeobjargproc)SequenceLongObject_sq_ass_item, + .sq_contains = (objobjproc)SequenceLongObject_sq_contains, + .sq_inplace_concat = (binaryfunc)NULL, // Not implemented. See above. + .sq_inplace_repeat = (ssizeargfunc)NULL, // Not implemented. See above. + }; + +And the ``SequenceLongObjectType`` type is declared with the ``tp_as_sequence`` field referring to this table: + +.. code-block:: c + + static PyTypeObject SequenceLongObjectType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceLongObject", + .tp_basicsize = sizeof(SequenceLongObject), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceLongObject_dealloc, + .tp_as_sequence = &SequenceLongObject_sequence_methods, + .tp_str = (reprfunc) SequenceLongObject___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "Sequence of long integers.", + .tp_methods = SequenceLongObject_methods, + .tp_init = (initproc) SequenceLongObject_init, + .tp_new = SequenceLongObject_new, + }; + +--------------------------- +Module Initialisation +--------------------------- + +This is as usual: + +.. code-block:: c + + static PyModuleDef sequence_object_cmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cSeqObject", + .m_doc = ( + "Example module that creates an extension type with sequence methods" + ), + .m_size = -1, + .m_methods = cIterator_methods, + }; + + PyMODINIT_FUNC + PyInit_cSeqObject(void) { + PyObject *m; + m = PyModule_Create(&sequence_object_cmodule); + if (m == NULL) { + return NULL; + } + + if (PyType_Ready(&SequenceLongObjectType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&SequenceLongObjectType); + if (PyModule_AddObject( + m, + "SequenceLongObject", + (PyObject *) &SequenceLongObjectType) < 0 + ) { + Py_DECREF(&SequenceLongObjectType); + Py_DECREF(m); + return NULL; + } + return m; + } + +==================================== +TODOs: +==================================== + +.. todo:: + + "Creating New Types": Add a section on making an object act like a number using + `tp_as_number `_. + See also `Number Object Structures `_ + +.. todo:: + + "Creating New Types": Add a section on making an object act like a mapping object using + `tp_as_mapping `_. + See also `Mapping Object Structures `_ + + NOTE: This is rarely used in the Python stdlib, really only for dictionaries. diff --git a/doc/sphinx/source/parsing_arguments.rst b/doc/sphinx/source/parsing_arguments.rst index 6e255cb..313559c 100644 --- a/doc/sphinx/source/parsing_arguments.rst +++ b/doc/sphinx/source/parsing_arguments.rst @@ -1,55 +1,245 @@ .. highlight:: python - :linenothreshold: 10 + :linenothreshold: 20 .. toctree:: :maxdepth: 3 -================================= -Parsing Python Arguments -================================= +.. Links to the Python documentation -This section describes how you write functions that accept Python ``*args`` and ``**kwargs`` arguments and how to extract ``PyObject`` or C fundamental types from them. +.. _PyArg_ParseTuple(): https://docs.python.org/3/c-api/arg.html#c.PyArg_ParseTuple +.. _PyArg_ParseTupleAndKeywords(): https://docs.python.org/3/c-api/arg.html#c.PyArg_ParseTupleAndKeywords +.. index:: + single: Parsing Arguments + +*************************** +Parsing Python Arguments +*************************** + +This section describes how you write functions that accept Python ``*args`` and ``**kwargs`` +arguments and how to extract ``PyObject`` or C fundamental types from them. + + +==================================== +Specifying the Function Arguments +==================================== + +Python has a myriad of ways of function arguments and the C API reflects that. +Two important features of CPython C functions are: + +- Declaring them correctly with the right signature and flags. +- Parsing the arguments and checking their types. + +These are described below. + +------------------------------- +C Function Declaration +------------------------------- + +The C function signature must be declared correctly depending on the arguments it is expected to work with. +Here is a small summary of the required declaration. + +In all cases the function has a *context* which is the first argument. + +- For free functions the first argument is the module within which the function is declared. + This allows you to access other attributes or functions in the module. +- For member functions (methods) the first argument is the object within which the function is declared ( ``self`` ). + This allows you to access other properties or functions in the object. + +If the argument is unused then the `Py_UNUSED `_ can be used to +supress a compiler warning or error thus: + +.. code-block:: c + + static PyObject * + parse_args_kwargs(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs); + +.. index:: + single: Parsing Arguments; ml_flags + +Setting the ``ml_flags`` Field +------------------------------ + +The `ml_flags `_ field in +`PyMethodDef `_ specifies the form of the arguments. + +.. index:: + single: Parsing Arguments; No Arguments + single: Parsing Arguments; METH_NOARGS + +No Arguments +^^^^^^^^^^^^^^^^^^ + +- The flags will be `METH_NOARGS `_ +- The C Function Signature will be ``PyObject *PyCFunction(PyObject *self, PyObject *args);`` +- The second argument will be ``NULL``. + +.. index:: + single: Parsing Arguments; One Argument + single: Parsing Arguments; METH_O + +One Argument +^^^^^^^^^^^^^^^^^^ + +- The flags will be `METH_O `_ +- The C Function Signature will be ``PyObject *PyCFunction(PyObject *self, PyObject *args);`` +- The second argument will be the single argument. + +.. index:: + single: Parsing Arguments; Multiple Arguments + single: Parsing Arguments; METH_VARARGS + +Multiple Positional Arguments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- The flags will be `METH_VARARGS `_ +- The C Function Signature will be ``PyObject *PyCFunction(PyObject *self, PyObject *args);`` +- Second value will be a tuple of arguments. +- `PyArg_ParseTuple()`_ is used to unpack the arguments. + +.. index:: + single: Parsing Arguments; Positional and Keyword Arguments + single: Parsing Arguments; METH_KEYWORDS + +Positional and Keyword Arguments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- The flags will be `METH_NOARGS | METH_KEYWORDS `_ +- The C Function Signature will be ``PyObject *PyCFunctionWithKeywords(PyObject *self, PyObject *args, PyObject *kwargs);`` +- Second value will be a sequence of arguments, the third the dictionary of arguments. +- `PyArg_ParseTupleAndKeywords()`_ is used to unpack the arguments. + +Documentation: + +- No arguments, single or multiple arguments: + `PyCFunction `_ +- Multiple argument and keywords `PyCFunctionWithKeywords `_ + +.. note:: + + I don't cover the use of `METH_FASTCALL `_ + in this tutorial. + +.. note:: + + `METH_KEYWORDS `_ + can only be used in combination with other flags. + +Example +^^^^^^^^ + +A function that takes positional and keyword arguments would be declared as: + +.. code-block:: c + + static PyObject * + parse_args_kwargs(PyObject *module, PyObject *args, PyObject *kwargs); + +And this would be added to the module, say, by using: + +.. code-block:: c + + static PyMethodDef cParseArgs_methods[] = { + /* ... */ + { + "parse_args_kwargs", + (PyCFunction) parse_args_kwargs, + METH_VARARGS | METH_KEYWORDS, + "Documentation for parse_args_kwargs()." + }, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +Parsing the Arguments +------------------------------ + +Once we have the C function correctly declared then the arguments have to parsed according to their types and, +if required, converted to C types (so-called "unboxing"). +This is done using the `PyArg_ParseTuple()`_ and `PyArg_ParseTupleAndKeywords()`_ +(ignoring “old-style” functions which use `PyArg_Parse `_). + +These use formatting strings that can become bewilderingly complex so this tutorial uses examples as an introduction. +The reference documentation is excellent: `argument parsing and building values `_. + +.. warning:: + + Error messages can be slightly misleading as the argument index is the *C* index + not the *Python* index. + + For example a C function using ``METH_VARARGS`` declared as: + + ``static PyObject *parse_args(PyObject *module, PyObject *args);`` + + Which expects the Python args[0] to be a bytes object and the Python args[1] + to be an integer by using: + + ``PyArg_ParseTuple(args, "Si", &arg0, &arg1)`` + + Calling this from Python with ``parse_args(21, 22)`` will give: + + ``TypeError: argument 1 must be bytes, not int``. + + The "1" here refers to the C index not the Python index. + The correct call would be to change the 0th Python argument to: + + ``cParseArgs.parse_args(b'21', 22)`` + + Also consider the signature: + + ``def parse_args(a: bytes, b: int, c: str = '') -> int:`` + + Called with ``parse_args(b'bytes', '456')`` gives the error: + + ``"'str' object cannot be interpreted as an integer"`` + + without specifying which argument it is referring to. + + +Examples +==================================== + +These examples are in ``src/cpy/cParseArgs.c`` and their tests are in ``tests/unit/test_c_parse_args.py``. + +.. index:: + single: Parsing Arguments Example; No Arguments + single: Parsing Arguments; METH_NOARGS ------------------------------------- No Arguments ------------------------------------ -The simplest from is a global function in a module that takes no arguments at all: +The simplest form is a global function in a module that takes no arguments at all: .. code-block:: c - static PyObject *_parse_no_args(PyObject *module) { - PyObject *ret = NULL; - + static PyObject *parse_no_args(PyObject *Py_UNUSED(module)) { + /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); - assert(ret); - goto finally; - except: - Py_XDECREF(ret); - ret = NULL; - finally: - return ret; + + Py_RETURN_NONE; } -This function is added to the module methods with the ``METH_NOARGS`` value. The Python interpreter will raise a ``TypeError`` in any arguments are offered. +This function is added to the module methods with the ``METH_NOARGS`` value. +The Python interpreter will raise a ``TypeError`` on any arguments are offered to the function. .. code-block:: c static PyMethodDef cParseArgs_methods[] = { /* Other functions here... */ - {"argsNone", (PyCFunction)_parse_no_args, METH_NOARGS, + { + "parse_no_args", + (PyCFunction)parse_no_args, + METH_NOARGS, "No arguments." }, /* Other functions here... */ {NULL, NULL, 0, NULL} /* Sentinel */ }; ------------------------------------- +.. index:: + single: Parsing Arguments Example; One Argument + single: Parsing Arguments; METH_O + One Argument ------------------------------------ @@ -57,32 +247,18 @@ There is no parsing required here, a single ``PyObject`` is expected: .. code-block:: c - static PyObject *_parse_one_arg(PyObject *module, - PyObject *arg - ) { - PyObject *ret = NULL; - assert(arg); + static PyObject *parse_one_arg(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) { /* arg as a borrowed reference and the general rule is that you Py_INCREF them - * whilst you have an interest in them. We do _not_ do that here for reasons - * explained below. + * whilst you have an interest in them. */ // Py_INCREF(arg); - + /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); - assert(ret); - goto finally; - except: - Py_XDECREF(ret); - ret = NULL; - finally: + /* If we were to treat arg as a borrowed reference and had Py_INCREF'd above we * should do this. See below. */ // Py_DECREF(arg); - return ret; + Py_RETURN_NONE; } This function can be added to the module with the ``METH_O`` flag: @@ -91,24 +267,28 @@ This function can be added to the module with the ``METH_O`` flag: static PyMethodDef cParseArgs_methods[] = { /* Other functions here... */ - {"argsOne", (PyCFunction)_parse_one_arg, METH_O, + { + "parse_one_arg", + (PyCFunction) parse_one_arg, + METH_O, "One argument." }, /* Other functions here... */ {NULL, NULL, 0, NULL} /* Sentinel */ }; -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Arguments as Borrowed References -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There is some subtlety here as indicated by the comments. ``*arg`` is not our reference, it is a borrowed reference so why don't we increment it at the beginning of this function and decrement it at the end? After all we are trying to protect against calling into some malicious/badly written code that could hurt us. For example: +There is some subtlety here as indicated by the comments. ``*arg`` is not our reference, it is a borrowed reference +so why don't we increment it at the beginning of this function and decrement it at the end? +After all we are trying to protect against calling into some malicious/badly written code that could hurt us. +For example: .. code-block:: c - static PyObject *foo(PyObject *module, - PyObject *arg - ) { + static PyObject *foo(PyObject *module, PyObject *arg) { /* arg has a minimum recount of 1. */ call_malicious_code_that_decrefs_by_one_this_argument(arg); /* arg potentially could have had a ref count of 0 and been deallocated. */ @@ -116,13 +296,13 @@ There is some subtlety here as indicated by the comments. ``*arg`` is not our re /* So now doing something with arg could be undefined. */ } -A solution would be, since ``arg`` is a 'borrowed' reference and borrowed references should always be incremented whilst in use and decremented when done with. This would suggest the following: +A solution would be, since ``arg`` is a 'borrowed' reference and borrowed references should always be incremented +whilst in use and decremented when done with. +This would suggest the following: .. code-block:: c - static PyObject *foo(PyObject *module, - PyObject *arg - ) { + static PyObject *foo(PyObject *module, PyObject *arg) { /* arg has a minimum recount of 1. */ Py_INCREF(arg); /* arg now has a minimum recount of 2. */ @@ -134,48 +314,49 @@ A solution would be, since ``arg`` is a 'borrowed' reference and borrowed refere /* But now arg could have had a ref count of 0 so is unsafe to use by the caller. */ } -But now we have just pushed the burden onto our caller. They created ``arg`` and passed it to us in good faith and whilst we have protected ourselves have not protected the caller and they can fail unexpectedly. So it is best to fail fast, an near the error site, that dastardly ``call_malicious_code_that_decrefs_by_one_this_argument()``. +But now we have just pushed the burden onto our caller. +They created ``arg`` and passed it to us in good faith and whilst we have protected ourselves have not protected the +caller and they can fail unexpectedly. +So it is best to fail fast, an near the error site, that dastardly +``call_malicious_code_that_decrefs_by_one_this_argument()``. Side note: Of course this does not protect you from malicious/badly written code that decrements by more than one :-) ----------------------------------------------------- +.. index:: + single: Parsing Arguments Example; Variable Number of Arguments + single: Parsing Arguments; METH_VARARGS + Variable Number of Arguments ---------------------------------------------------- -The function will be called with two arguments, the module and a ``PyListObject`` that contains a list of arguments. You can either parse this list yourself or use a helper method to parse it into Python and C types. +The function will be called with two arguments, the module and a ``PyTupleObject`` that contains a tuple of arguments. +You can either parse this list yourself or use a helper method to parse it into Python and C types. -In the following code we are expecting a string, an integer and an optional integer whose default value is 8. In Python the equivalent function declaration would be:: +In the following code we are expecting a bytes object, an integer and an optional string whose default value is +'default_string'. +For demonstration purposes, this returns the same three arguments. +In Python the equivalent function signature would be:: - def argsOnly(theString, theInt, theOptInt=8): + def parse_args(a: bytes, b: int, c: str = 'default_string') \ + -> typing.Tuple[bytes, int, str]: -Here is the C code, note the string that describes the argument types passed to ``PyArg_ParseTuple``, if these types are not present a ``ValueError`` will be set. +Here is the C code, note the string that describes the argument types passed to ``PyArg_ParseTuple``, if these types +are not present a ``ValueError`` will be set. .. code-block:: c - static PyObject *_parse_args(PyObject *module, - PyObject *args - ) { - PyObject *ret = NULL; - PyObject *pyStr = NULL; - int arg1, arg2; - - arg2 = 8; /* Default value. */ - if (! PyArg_ParseTuple(args, "Si|i", &pyStr, &arg1, &arg2)) { - goto except; + static PyObject *parse_args(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *arg_0 = NULL; + int arg_1; + char *arg_2 = "default_string"; + + if (!PyArg_ParseTuple(args, "Si|s", &arg_0, &arg_1, &arg_2)) { + return NULL; } - + /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); - assert(ret); - goto finally; - except: - Py_XDECREF(ret); - ret = NULL; - finally: - return ret; + + return Py_BuildValue("Ois", arg_0, arg_1, arg_2); } This function can be added to the module with the ``METH_VARARGS`` flag: @@ -184,114 +365,120 @@ This function can be added to the module with the ``METH_VARARGS`` flag: static PyMethodDef cParseArgs_methods[] = { /* Other functions here... */ - {"argsOnly", (PyCFunction)_parse_args, METH_VARARGS, - "Reads args only." + { + "parse_args", + (PyCFunction) parse_args, + METH_VARARGS, + "parse_args() documentation" }, /* Other functions here... */ {NULL, NULL, 0, NULL} /* Sentinel */ }; --------------------------------------------------------------------------- +This code can be seen in ``src/cpy/cParseArgs.c``. +It is tested in ``tests.unit.test_c_parse_args.test_parse_args``. +Failure modes, when the wrong arguments are passed are tested in ``tests.unit.test_c_parse_args.test_parse_args_raises``. +Note the wide variety of error messages that are obtained. + +.. index:: + single: Parsing Arguments Example; Variable Number of Arguments + single: Parsing Arguments Example; Keyword Arguments + single: Parsing Arguments; METH_KEYWORDS + Variable Number of Arguments and Keyword Arguments -------------------------------------------------------------------------- -The function will be called with two arguments, the module, a ``PyListObject`` that contains a list of arguments and a ``PyDictObject`` that contains a dictionary of keyword arguments. You can either parse these yourself or use a helper method to parse it into Python and C types. +The function will be called with three arguments, the module, a ``PyTupleObject`` that contains a tuple of arguments +and a ``PyDictObject`` that contains a dictionary of keyword arguments. +The keyword arguments can be NULL if there are no keyword arguments. +You can either parse these yourself or use a helper method to parse it into Python and C types. -In the following code we are expecting a string, an integer and an optional integer whose default value is 8. In Python the equivalent function declaration would be:: +In the following code we are expecting a sequence and an optional integer count defaulting to 1. +It returns the `sequence` repeated `count` times. +In Python the equivalent function declaration would be:: - def argsKwargs(theString, theOptInt=8): + def parse_args_kwargs(sequence=typing.Sequence[typing.Any], count: = 1) \ + -> typing.Sequence[typing.Any]: + return sequence * count -Here is the C code, note the string that describes the argument types passed to ``PyArg_ParseTuple``, if these types are not present a ``ValueError`` will be set. +Here is the C code, note the string ``"O|i"`` that describes the argument types passed to +``PyArg_ParseTupleAndKeywords``, if these types are not present a ``TypeError`` will be set. .. code-block:: c - static PyObject *_parse_args_kwargs(PyObject *module, - PyObject *args, - PyObject *kwargs - ) { + static PyObject * + parse_args_kwargs(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { PyObject *ret = NULL; - PyObject *pyStr = NULL; - int arg2; + PyObject *py_sequence = NULL; + int count = 1; /* Default. */ static char *kwlist[] = { - "theString", - "theOptInt", - NULL + "sequence", /* A sequence object, str, list, tuple etc. */ + "count", /* Python int converted to a C int. */ + NULL, }; - - /* If you are interested this is a way that you can trace the input. - PyObject_Print(module, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(args, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(kwargs, stdout, 0); - fprintf(stdout, "\n"); - * End trace */ - - arg2 = 8; /* Default value. */ - if (! PyArg_ParseTupleAndKeywords(args, kwargs, "S|i", - kwlist, &pyStr, &arg2)) { + + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, "O|i", kwlist, &py_sequence, &count + )) { goto except; } - + /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); - assert(ret); + + ret = PySequence_Repeat(py_sequence, count); + if (ret == NULL) { + goto except; + } + assert(!PyErr_Occurred()); goto finally; except: + assert(PyErr_Occurred()); Py_XDECREF(ret); ret = NULL; finally: return ret; } - -This function can be added to the module with the ``METH_VARARGS`` and ``METH_KEYWORDS`` flags: +This function can be added to the module with both the ``METH_VARARGS`` and ``METH_KEYWORDS`` flags set: .. code-block:: c static PyMethodDef cParseArgs_methods[] = { /* Other functions here... */ - {"argsKwargs", (PyCFunction)_parse_args_kwargs, + { + "parse_args_kwargs", + (PyCFunction) parse_args_kwargs, METH_VARARGS | METH_KEYWORDS, - _parse_args_kwargs_docstring + "parse_args_kwargs() documentation" }, /* Other functions here... */ {NULL, NULL, 0, NULL} /* Sentinel */ }; +This code can be seen in ``src/cpy/cParseArgs.c``. +It is tested in ``tests.unit.test_c_parse_args.test_parse_args_kwargs`` which shows the variety of ways this can be +called. +Failure modes, when the wrong arguments are passed are tested in +``tests.unit.test_c_parse_args.test_parse_args_kwargs_raises``. + All arguments are keyword arguments so this function can be called in a number of ways, all of the following are equivalent: .. code-block:: python - - argsKwargs('foo') - argsKwargs('foo', 8) - argsKwargs(theString='foo') - argsKwargs(theOptInt=8, theString='foo') - argsKwargs(theString, theOptInt=8) -If you want the function signature to be ``argsKwargs(theString, theOptInt=8)`` with a single argument and a single optional keyword argument then put an empty string in the kwlist array: + cParseArgs.parse_args_kwargs([1, 2, 3], 2) + cParseArgs.parse_args_kwargs([1, 2, 3], count=2) + cParseArgs.parse_args_kwargs(sequence=[1, 2, 3], count=2) -.. code-block:: c - - /* ... */ - static char *kwlist[] = { - "", - "theOptInt", - NULL - }; - /* ... */ +.. index:: + single: Parsing Arguments Example; Keyword Arguments and C++11 -.. note:: - If you use ``|`` in the parser format string you have to set the default values for those optional arguments yourself in the C code. This is pretty straightforward if they are fundamental C types as ``arg2 = 8`` above. For Python values is a bit more tricky as described next. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Keyword Arguments and C++11 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -C++11 compilers warn when creating non-const ``char*`` from string literals as we have done with the keyword array above. The solution is to declare these ``const char*`` however ``PyArg_ParseTupleAndKeywords`` expects a ``char **``. The solution is to cast away const in the call: +C++11 compilers warn when creating non-const ``char*`` from string literals as we have done with the keyword array +above. +The solution is to declare these ``const char*`` however `PyArg_ParseTupleAndKeywords()`_ expects a ``char **``. +The solution is to cast away const in the call: .. code-block:: c @@ -305,125 +492,357 @@ C++11 compilers warn when creating non-const ``char*`` from string literals as w /* ... */ -.. _cpython_default_arguments: +.. index:: + single: Parsing Arguments Example; Default String Arguments + single: Parsing Arguments Example; Default Bytes Arguments + single: Default Arguments; C + single: Py_buffer --------------------------------------------------------------------------- -Being Pythonic with Default Arguments --------------------------------------------------------------------------- +Default String and Bytes Arguments +------------------------------------------ + +The recommended way to accept binary data is to parse the argument using the ``"y*"`` formatting string and supply +a `Py_Buffer `_ argument. +This also applies to strings, using the ``"s*"`` formatting string, where they might contain ``'\0'`` characters. + +Typically this would be done with C code such as this: + +.. code-block:: c + + Py_buffer arg; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*", kwlist, &arg)) { + assert(PyErr_Occurred()); + return NULL; + } + /* arg.buf has the byte data of length arg.len */ + +However this will likely segfault if it is used as a default argument using ``"|y*"`` formatting string as the +``Py_Buffer`` is uninitialized. +Here is the fix for using a default value of ``b''``: + +.. code-block:: c + + Py_buffer arg; + arg.buf = NULL; + arg.len = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|y*", kwlist, &arg)) { + assert(PyErr_Occurred()); + return NULL; + } -If the arguments default to some C fundamental type the code above is fine. However if the arguments default to Python objects then a little more work is needed. Here is a function that has a tuple and a dict as default arguments, in other words the Python signature: + +For a different default value, say ``b'default'``, then this will work. +The Python signature is: .. code-block:: python - def function(arg_0=(42, "this"), arg_1={}): + def parse_default_bytes_object(b: bytes = b"default") -> bytes: + +The complete C code is: + +.. code-block:: c + + static PyObject * + parse_default_bytes_object(PyObject *Py_UNUSED(module), PyObject *args, + PyObject *kwargs) { + static const char *arg_default = "default"; + Py_buffer arg; + arg.buf = (void *)arg_default; + arg.len = strlen(arg_default); + static char *kwlist[] = { + "b", + NULL, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|y*", kwlist, &arg)) { + assert(PyErr_Occurred()); + return NULL; + } + return Py_BuildValue("y#", arg.buf, arg.len); + } + +See ``tests/unit/test_c_parse_args.py`` for some Python uses of this code. -The first argument is immutable, the second is mutable and so we need to mimic the well known behaviour of Python with mutable arguments. Mutable default arguments are evaluated once only at function definition time and then becomes a (mutable) property of the function. For example: +.. index:: + single: Positional Only Arguments + single: Keyword Only Arguments + single: Parsing Arguments Example; Positional Only Arguments + single: Parsing Arguments Example; Keyword Only Arguments + +Positional Only and Keyword Only Arguments +----------------------------------------------- + +This section shows how to achieve +`positional only `_ +and `keyword only `_ arguments in a C extension. +These are described in the Python documentation for +`Special parameters `_ +Specifically `positional only parameters `_ +and `keyword only arguments `_. + +Suppose we want the functional equivalent of the Python function signature +(reproducing https://docs.python.org/3/tutorial/controlflow.html#special-parameters ): .. code-block:: python - >>> def f(l=[]): - ... l.append(9) - ... print(l) - ... - >>> f() - [9] - >>> f() - [9, 9] - >>> f([]) - [9] - >>> f() - [9, 9, 9] - -In C we can get this behaviour by treating the mutable argument as ``static``, the immutable argument does not need to be ``static`` but it will do no harm if it is (if non-``static`` it will have to be initialised on every function call). + def parse_pos_only_kwd_only(pos1: str, pos2: int, /, pos_or_kwd: bytes, *, + kwd1: float = 256.0, kwd2: int = -421): + return None -My advice: Always make all ``PyObject*`` references to default arguments ``static``. +This is achieved by combining two techniques: -So first we declare a ``static PyObject*`` for each default argument: +- Positional only: The strings in the ``*kwlist`` passed to `PyArg_ParseTupleAndKeywords()`_ are empty. +- Keyword only: The formatting string passed to `PyArg_ParseTupleAndKeywords()`_ uses the ``'$'`` character. + +A function using either positional only or keyword only arguments must use the flags ``METH_VARARGS | METH_KEYWORDS`` +and uses `PyArg_ParseTupleAndKeywords()`_. Currently, all keyword-only arguments must also be optional arguments, so ``'|'`` must always be +specified before ``'$'`` in the format string. + +Here is the C code: .. code-block:: c - static PyObject *_parse_args_with_python_defaults(PyObject *module, PyObject *args) { - PyObject *ret = NULL; - - /* This first pointer need not be static as the argument is immutable - * but if non-static must be NULL otherwise the following code will be undefined. - */ - static PyObject *pyObjDefaultArg_0; - static PyObject *pyObjDefaultArg_1; /* Must be static if mutable. */ + static PyObject * + parse_pos_only_kwd_only(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + /* Arguments, first three are required. */ + Py_buffer pos1; + int pos2; + Py_buffer pos_or_kwd; + /* Last two are optional. */ + double kwd1 = 256.0; + int kwd2 = -421; + static char *kwlist[] = { + "", /* pos1 is positional only. */ + "", /* pos2 is positional only. */ + "pos_or_kwd", /* pos_or_kwd can be positional or keyword argument. */ + /* NOTE: As '$' is in format string the next to are keyword only. */ + "kwd1", /* kwd1 is keyword only argument. */ + "kwd2", /* kwd2 is keyword only argument. */ + NULL, + }; -Then we declare a ``PyObject*`` for each argument that will either reference the default or the passed in argument. It is important that these ``pyObjArg_...`` pointers are NULL so that we can subsequently detect if ``PyArg_ParseTuple`` has set them non-``NULL``. + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*iy*|$di", kwlist, &pos1, &pos2, + &pos_or_kwd, &kwd1, &kwd2)) { + assert(PyErr_Occurred()); + return NULL; + } + /* Return the parsed arguments. + * NOTE the asymmetry between "s*iy*|$di" above and "s#iy#di" here. */ + return Py_BuildValue("s#iy#di", pos1.buf, pos1.len, pos2, pos_or_kwd.buf, + pos_or_kwd.len, kwd1, kwd2); + } -.. code-block:: c - /* These 'working' pointers are the ones we use in the body of the function - * They either reference the supplied argument or the default (static) argument. - * We must treat these as "borrowed" references and so must incref them - * while they are in use then decref them when we exit the function. - */ - PyObject *pyObjArg_0 = NULL; - PyObject *pyObjArg_1 = NULL; - -Then, if the default values have not been initialised, initialise them. In this case it is a bit tedious merely because of the nature of the arguments. So in practice this might be clearer if this was in separate function: +.. index:: + single: Parsing Arguments Example; Functional Conversion to C + +Parsing Arguments With Functional Conversion to C +--------------------------------------------------------- + +Often you want to convert a Python argument to a C value(s) in a way that is not covered by the format strings +provided by the Python C API. To do this you can provide a special conversion function in C and give it to +`PyArg_ParseTuple()`_ or `PyArg_ParseTupleAndKeywords()`_. + +In this example we are expecting the Python argument to be a list of integers and we want the sum of them as +a C ``long``. First create a C function that takes a Python list, checks it and sums the values. +The function returns 1 on success or 0 on error and, in that case, and exception is expected to be set. +On success the result will be written into the opaque pointer, here called ``address``: .. code-block:: c - /* Initialise first argument to its default Python value. */ - if (! pyObjDefaultArg_0) { - pyObjDefaultArg_0 = PyTuple_New(2); - if (! pyObjDefaultArg_0) { - PyErr_SetString(PyExc_RuntimeError, "Can not create tuple!"); - goto except; - } - if(PyTuple_SetItem(pyObjDefaultArg_0, 0, PyLong_FromLong(42))) { - PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[0]!"); - goto except; - } - if(PyTuple_SetItem(pyObjDefaultArg_0, 1, PyUnicode_FromString("This"))) { - PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[1]!"); - goto except; - } + int sum_list_of_longs(PyObject *list_longs, void *address) { + PyObject *item = NULL; + + /* Note: PyList_Check allows sub-types. */ + if (! list_longs || ! PyList_Check(list_longs)) { + PyErr_Format(PyExc_TypeError, + "check_list_of_longs(): First argument is not a list" + ); + return 0; } - /* Now the second argument. */ - if (! pyObjDefaultArg_1) { - pyObjDefaultArg_1 = PyDict_New(); + long result = 0L; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list_longs); ++i) { + item = PyList_GetItem(list_longs, i); + if (!PyLong_CheckExact(item)) { + PyErr_Format(PyExc_TypeError, + "check_list_of_longs(): Item %d is not a Python integer.", i + ); + return 0; + } + /* PyLong_AsLong() must always succeed because of check above. */ + result += PyLong_AsLong(item); } - + long *p_long = (long *) address; + *p_long = result; + return 1; /* Success. */ + } + +Now we can pass this function to ``PyArg_ParseTuple`` with the ``"O&"`` formatting string that takes two arguments, +the Python list and the C conversion function. On success ``PyArg_ParseTuple`` writes the value to the target, +``result``. + +In this case the function just returns the sum of the integers in the list. +Here is the C code. -Now parse the given arguments to see what, if anything, is there. ``PyArg_ParseTuple`` will set each working pointer non-``NULL`` if the argument is present. As we set the working pointers ``NULL`` prior to this call we can now tell if any argument is present. - .. code-block:: c - if (! PyArg_ParseTuple(args, "|OO", &pyObjArg_0, &pyObjArg_1)) { + static PyObject * + parse_args_with_function_conversion_to_c(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *ret = NULL; + long result; + + if (!PyArg_ParseTuple(args, "O&", sum_list_of_longs, &result)) { + /* NOTE: If check_list_of_numbers() returns 0 an error should be set. */ + assert(PyErr_Occurred()); + goto except; + } + + /* Your code here...*/ + ret = PyLong_FromLong(result); + if (ret == NULL) { goto except; } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; + } + +.. index:: + single: Parsing Arguments Example; Default Mutable Arguments + single: Default Arguments, Mutable; C + +.. _cpython_default_mutable_arguments: + +Being Pythonic with Default Mutable Arguments +============================================= + +If the arguments default to some C fundamental type the code above is fine. +However if the mutable arguments default to Python objects then a little more work is needed. + +.. note:: + + See :ref:`cpp_and_cpython.handling_default_arguments` for a way of simplifying this with C++. + +Here is a function that has an object and as default argument as a list, appends the object to the list and +returns the list. +The Python signature is: + +.. code-block:: python + + def function(arg_0, arg_1=[]): + arg_1.append(arg_0) + return arg_1 + +The first argument is immutable, the second is mutable. +We need to mimic the well known behaviour of Python with mutable arguments where default arguments are evaluated once +only at function definition time and then becomes a (mutable) property of the function. -Now switch our working pointers to the default argument if no argument is given. We also treat these as "borrowed" references regardless of whether they are default or supplied so increment the refcount (we must decrement the refcount when done). +For example: + +.. code-block:: python + + >>> function(1) + [1] + >>> function(2) + [1, 2] + >>> my_list = [] + >>> f(10, my_list) + [10] + >>> f(10) + [1, 2, 10] + +In C we can get this behaviour by treating the mutable argument as ``static``, the immutable argument does not need to +be ``static`` but it will do no harm if it is (if non-``static`` it will have to be initialised on every function call). + +My advice: Always make all ``PyObject*`` references to default arguments ``static``. + +So first we declare a ``static PyObject*`` for each default argument: .. code-block:: c - /* First argument. */ - if (! pyObjArg_0) { - pyObjArg_0 = pyObjDefaultArg_0; - } - Py_INCREF(pyObjArg_0); - - /* Second argument. */ - if (! pyObjArg_1) { - pyObjArg_1 = pyObjDefaultArg_1; + /** Parse the args where we are simulating mutable default of an empty list. + * + * This is equivalent to: + * + * def parse_args_with_mutable_defaults_macro_helper(obj, default_list=[]): + * default_list.append(obj) + * return default_list + * + * See also parse_args_with_mutable_defaults_macro_helper() in cParseArgsHelper.cpp + * This adds the object to the list and returns None. + * + * This imitates the Python way of handling defaults. + */ + static PyObject *parse_args_with_mutable_defaults(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + /* Pointers to the default argument, initialised below. */ + static PyObject *arg_1_default = NULL; + /* Set defaults for argument 1. */ + if (!arg_1_default) { + arg_1_default = PyList_New(0); } - Py_INCREF(pyObjArg_1); -Now write the main body of your function and that must be followed by this clean up code: +Then we declare a ``PyObject*`` for each argument that will either reference the default or the passed in argument. +It is important that these ``pyObjArg_...`` pointers are NULL so that we can subsequently detect if +``PyArg_ParseTuple`` has set them non-``NULL``. .. code-block:: c - /* Your code here using pyObjArg_0 and pyObjArg_1 ...*/ - - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); - assert(ret); - goto finally; + /* This pointer is the one we use in the body of the function, it + * either points at the supplied argument or the default (static) argument. + */ + PyObject *arg_1 = NULL; + + +Now parse the given arguments to see what, if anything, is there. +``PyArg_ParseTuple`` will set each working pointer non-``NULL`` if the argument is present. +As we set the working pointers ``NULL`` prior to this call we can now tell if any argument is present. + +.. code-block:: c + + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &arg_1)) { + goto except; + } + + +Now switch our working pointers to the default argument if no argument is given. +We also treat these as "borrowed" references regardless of whether they are default or supplied so increment the +refcount (we must decrement the refcount when done). + +.. code-block:: c + + /* If optional argument absent then switch to defaults. */ + if (!arg_1) { + arg_1 = arg_1_default; + } + +Now write the main body of your function and that must be followed by this clean up code: + +.. code-block:: c + + /* Your code here...*/ + + /* Append the first argument to the second. + * PyList_Append increments the reference count of arg_0. */ + if (PyList_Append(arg_1, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; + } + /* Success. */ + assert(!PyErr_Occurred()); + /* Increments the default or the given argument. */ + Py_INCREF(arg_1); + ret = arg_1; + goto finally; Now the two blocks ``except`` and ``finally``. @@ -434,143 +853,187 @@ Now the two blocks ``except`` and ``finally``. Py_XDECREF(ret); ret = NULL; finally: - /* Decrement refcount to match the increment above. */ - Py_XDECREF(pyObjArg_0); - Py_XDECREF(pyObjArg_1); return ret; } -An important point here is the use of ``Py_XDECREF`` in the ``finally:`` block, we can get here through a number of paths, including through the ``except:`` block and in some cases the ``pyObjArg_...`` will be ``NULL`` (for example if ``PyArg_ParseTuple`` fails). So ``Py_XDECREF`` it must be. - Here is the complete C code: .. code-block:: c :linenos: - static PyObject *_parse_args_with_python_defaults(PyObject *module, PyObject *args) { + /** Parse the args where we are simulating mutable default of an empty list. + * + * This is equivalent to: + * + * def parse_args_with_mutable_defaults_macro_helper(obj, default_list=[]): + * default_list.append(obj) + * return default_list + * + * See also parse_args_with_mutable_defaults_macro_helper() in cParseArgsHelper.cpp + * This adds the object to the list and returns None. + * + * This imitates the Python way of handling defaults. + */ + static PyObject *parse_args_with_mutable_defaults(PyObject *Py_UNUSED(module), + PyObject *args) { PyObject *ret = NULL; - static PyObject *pyObjDefaultArg_0; - static PyObject *pyObjDefaultArg_1; - PyObject *pyObjArg_0 = NULL; - PyObject *pyObjArg_1 = NULL; - - if (! pyObjDefaultArg_0) { - pyObjDefaultArg_0 = PyTuple_New(2); - if (! pyObjDefaultArg_0) { - PyErr_SetString(PyExc_RuntimeError, "Can not create tuple!"); - goto except; - } - if(PyTuple_SetItem(pyObjDefaultArg_0, 0, PyLong_FromLong(42))) { - PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[0]!"); - goto except; - } - if(PyTuple_SetItem(pyObjDefaultArg_0, 1, PyUnicode_FromString("This"))) { - PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[1]!"); - goto except; - } - } - if (! pyObjDefaultArg_1) { - pyObjDefaultArg_1 = PyDict_New(); + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + /* Pointers to the default argument, initialised below. */ + static PyObject *arg_1_default = NULL; + /* Set defaults for argument 1. */ + if (!arg_1_default) { + arg_1_default = PyList_New(0); } - - if (! PyArg_ParseTuple(args, "|OO", &pyObjArg_0, &pyObjArg_1)) { + /* This pointer is the one we use in the body of the function, it + * either points at the supplied argument or the default (static) argument. + */ + PyObject *arg_1 = NULL; + + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &arg_1)) { goto except; } - if (! pyObjArg_0) { - pyObjArg_0 = pyObjDefaultArg_0; - } - Py_INCREF(pyObjArg_0); - if (! pyObjArg_1) { - pyObjArg_1 = pyObjDefaultArg_1; + /* If optional argument absent then switch to defaults. */ + if (!arg_1) { + arg_1 = arg_1_default; } - Py_INCREF(pyObjArg_1); - + /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); - assert(ret); + + /* Append the first argument to the second. + * PyList_Append increments the reference count of arg_0. */ + if (PyList_Append(arg_1, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; + } + + /* Success. */ + assert(!PyErr_Occurred()); + /* Increments the default or the given argument. */ + Py_INCREF(arg_1); + ret = arg_1; goto finally; except: assert(PyErr_Occurred()); Py_XDECREF(ret); ret = NULL; finally: - Py_XDECREF(pyObjArg_0); - Py_XDECREF(pyObjArg_1); return ret; } -^^^^^^^^^^^^^^^^^^^^^^^^ -Simplifying Macros -^^^^^^^^^^^^^^^^^^^^^^^^ +The code can be found in ``src/cpy/ParseArgs/cParseArgs.c``. + +Tests are in ``test_parse_args_with_mutable_defaults()`` in ``tests/unit/test_c_parse_args.py`` + +.. index:: + single: Parsing Arguments Example; Helper Macros -For simple default values some macros may help. The first one declares and initialises the default value. It takes three arguments: +Helper Macros +============= -* The name of the argument variable, a static ``PyObject`` named ``default_`` will also be created. -* The default value which should return a new reference. -* The value to return on failure to create a default value, usually -1 or ``NULL``. +Some macros can make this easier. +These are in ``src/cpy/ParseArgs/cParseArgsHelper.cpp``. +Firstly a macro to declare the static default object: .. code-block:: c - #define PY_DEFAULT_ARGUMENT_INIT(name, value, ret) \ - PyObject *name = NULL; \ - static PyObject *default_##name = NULL; \ - if (! default_##name) { \ - default_##name = value; \ - if (! default_##name) { \ - PyErr_SetString(PyExc_RuntimeError, "Can not create default value for " #name); \ - return ret; \ - } \ + #define PY_DEFAULT_ARGUMENT_INIT(name, value, ret) \ + PyObject *name = NULL; \ + static PyObject *default_##name = NULL; \ + if (! default_##name) { \ + default_##name = value; \ + if (! default_##name) { \ + PyErr_SetString( \ + PyExc_RuntimeError, \ + "Can not create default value for " #name \ + ); \ + return ret; \ + } \ } -The second one assigns the argument to the default if it is not initialised and increments the reference count. It just takes the name of the argument: +.. warning:: + + When using this macro in a source file then make sure each given "name" argument is unique within the + translation unit. + +And a macro to set it: .. code-block:: c - #define PY_DEFAULT_ARGUMENT_SET(name) if (! name) name = default_##name; \ - Py_INCREF(name) + #define PY_DEFAULT_ARGUMENT_SET(name) \ + if (! name) { \ + name = default_##name; \ + } -And they can be used like this when implementing a Python function signature such as:: - - def do_something(self, encoding='utf-8', the_id=0, must_log=True): - # ... - return None +And a macro to check the type of the argument: + +.. code-block:: c + + #define PY_DEFAULT_CHECK(name, check_function, type) \ + if (!check_function(name)) { \ + PyErr_Format( \ + PyExc_TypeError, \ + #name " must be " #type ", not \"%s\"", \ + Py_TYPE(name)->tp_name \ + ); \ + return NULL; \ + } -Here is that function implemented in C: +.. index:: + single: Default Arguments, Immutable; C + +Immutable Arguments +------------------- + +These can be used thus. +This is equivalent to the Python function: + +.. code-block:: python + + def parse_defaults_with_helper_macro( + encoding_m: str = "utf-8", + the_id_m: int = 1024, + log_interval_m: float = 8.0): + return encoding_m, the_id_m, log_interval_m + +Here it is in C: .. code-block:: c - static PyObject* - do_something(something *self, PyObject *args, PyObject *kwds) { + static PyObject * + parse_defaults_with_helper_macro(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { PyObject *ret = NULL; /* Initialise default arguments. Note: these might cause an early return. */ - PY_DEFAULT_ARGUMENT_INIT(encoding, PyUnicode_FromString("utf-8"), NULL); - PY_DEFAULT_ARGUMENT_INIT(the_id, PyLong_FromLong(0L), NULL); - PY_DEFAULT_ARGUMENT_INIT(must_log, PyBool_FromLong(1L), NULL); - - static const char *kwlist[] = { "encoding", "the_id", "must_log", NULL }; - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|Oip", - const_cast(kwlist), - &encoding, &the_id, &must_log)) { - return NULL; + PY_DEFAULT_ARGUMENT_INIT(encoding_m, PyUnicode_FromString("utf-8"), NULL); + PY_DEFAULT_ARGUMENT_INIT(the_id_m, PyLong_FromLong(DEFAULT_ID), NULL); + PY_DEFAULT_ARGUMENT_INIT(log_interval_m, PyFloat_FromDouble(DEFAULT_FLOAT), NULL); + + static const char *kwlist[] = {"encoding", "the_id", "log_interval", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", + const_cast(kwlist), + &encoding_m, &the_id_m, &log_interval_m)) { + goto except; } - /* + /* * Assign absent arguments to defaults and increment the reference count. * Don't forget to decrement the reference count before returning! */ - PY_DEFAULT_ARGUMENT_SET(encoding); - PY_DEFAULT_ARGUMENT_SET(the_id); - PY_DEFAULT_ARGUMENT_SET(must_log); + PY_DEFAULT_ARGUMENT_SET(encoding_m); + PY_DEFAULT_ARGUMENT_SET(the_id_m); + PY_DEFAULT_ARGUMENT_SET(log_interval_m); + + /* Check the types of the given or default arguments. */ + PY_DEFAULT_CHECK(encoding_m, PyUnicode_Check, "str"); + PY_DEFAULT_CHECK(the_id_m, PyLong_Check, "int"); + PY_DEFAULT_CHECK(log_interval_m, PyFloat_Check, "float"); /* - * Use encoding, the_id, must_log from here on... + * Use 'encoding': Python str, 'the_id': C long, 'must_log': C long from here on... */ - Py_INCREF(Py_None); - ret = Py_None; - assert(! PyErr_Occurred()); + /* Py_BuildValue("O") increments the reference count. */ + ret = Py_BuildValue("OOO", encoding_m, the_id_m, log_interval_m); + assert(!PyErr_Occurred()); assert(ret); goto finally; except: @@ -578,99 +1041,148 @@ Here is that function implemented in C: Py_XDECREF(ret); ret = NULL; finally: - Py_DECREF(encoding); - Py_DECREF(the_id); - Py_DECREF(must_log); return ret; } -^^^^^^^^^^^^^^^^^^^^^^^^ -Simplifying C++11 class -^^^^^^^^^^^^^^^^^^^^^^^^ +The code is: -With C++ we can make this a bit smoother. We declare a class thus: +- Function: ``parse_defaults_with_helper_macro()`` +- Location: ``src/cpy/ParseArgs/cParseArgsHelper.cpp`` -.. code-block:: cpp +The tests are in ``tests/unit/test_c_parse_args_helper.py``. - /** Class to simplify default arguments. - * - * Usage: - * - * static DefaultArg arg_0(PyLong_FromLong(1L)); - * static DefaultArg arg_1(PyUnicode_FromString("Default string.")); - * if (! arg_0 || ! arg_1) { - * return NULL; - * } - * - * if (! PyArg_ParseTupleAndKeywords(args, kwargs, "...", - const_cast(kwlist), - &arg_0, &arg_1, ...)) { - return NULL; - } +If you are in a C++ environment then the section on :ref:`cpp_and_cpython.handling_default_arguments` can help. + +.. index:: + single: Default Arguments, Mutable; C + +Mutable Arguments +------------------- + +These macros also work with mutable arguments. + +The following C code emulates this Python code: + +.. code-block:: python + + def parse_mutable_defaults_with_helper_macro(obj, default_list=[]): + default_list.append(obj) + return default_list + +Here is the C code: + +.. code-block:: c + + /** Parse the args where we are simulating mutable default of an empty list. + * This uses the helper macros. * - * Then just use arg_0, arg_1 as if they were a PyObject* (possibly - * might need to be cast to some specific PyObject*). + * This is equivalent to: * - * WARN: This class is designed to be statically allocated. If allocated - * on the heap or stack it will leak memory. That could be fixed by - * implementing: + * def parse_mutable_defaults_with_helper_macro(obj, default_list=[]): + * default_list.append(obj) + * return default_list * - * ~DefaultArg() { Py_XDECREF(m_default); } + * This adds the object to the list and returns None. * - * But this will be highly dangerous when statically allocated as the - * destructor will be invoked with the Python interpreter in an - * uncertain state and will, most likely, segfault: - * "Python(39158,0x7fff78b66310) malloc: *** error for object 0x100511300: pointer being freed was not allocated" + * This imitates the Python way of handling defaults. */ - class DefaultArg { - public: - DefaultArg(PyObject *new_ref) : m_arg { NULL }, m_default { new_ref } {} - // Allow setting of the (optional) argument with - // PyArg_ParseTupleAndKeywords - PyObject **operator&() { m_arg = NULL; return &m_arg; } - // Access the argument or the default if default. - operator PyObject*() const { - return m_arg ? m_arg : m_default; - } - // Test if constructed successfully from the new reference. - explicit operator bool() { return m_default != NULL; } - protected: - PyObject *m_arg; - PyObject *m_default; - }; - + static PyObject *parse_mutable_defaults_with_helper_macro(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + /* Pointers to the default argument, initialised below. */ + /* Initialise default arguments. Note: these might cause an early return. */ + PY_DEFAULT_ARGUMENT_INIT(list_argument_m, PyList_New(0), NULL); -And we can use ``DefaultArg`` like this: + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &list_argument_m)) { + goto except; + } + /* If optional argument absent then switch to defaults. */ + PY_DEFAULT_ARGUMENT_SET(list_argument_m); + PY_DEFAULT_CHECK(list_argument_m, PyList_Check, "list"); -.. code-block:: c + /* Your code here...*/ - static PyObject* - do_something(something *self, PyObject *args, PyObject *kwds) { - PyObject *ret = NULL; - /* Initialise default arguments. */ - static DefaultArg encoding { PyUnicode_FromString("utf-8") }; - static DefaultArg the_id { PyLong_FromLong(0L) }; - static DefaultArg must_log { PyBool_FromLong(1L) }; - - /* Check that the defaults are non-NULL i.e. succesful. */ - if (!encoding || !the_id || !must_log) { - return NULL; - } - - static const char *kwlist[] = { "encoding", "the_id", "must_log", NULL }; - /* &encoding etc. accesses &m_arg in DefaultArg because of PyObject **operator&() */ - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|Oip", - const_cast(kwlist), - &encoding, &the_id, &must_log)) { - return NULL; + /* Append the first argument to the second. + * PyList_Append() increments the refcount of arg_0. */ + if (PyList_Append(list_argument_m, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; } - /* - * Use encoding, the_id, must_log from here on as PyObject* since we have - * operator PyObject*() const ... - * - * So if we have a function: - * set_encoding(PyObject *obj) { ... } - */ - set_encoding(encoding); - /* ... */ + + /* Success. */ + assert(!PyErr_Occurred()); + /* This increments the default or the given argument. */ + Py_INCREF(list_argument_m); + ret = list_argument_m; + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; } + +Here is some test code from ``tests/unit/test_c_parse_args_helper.py``. + +First a test to establish the Python behaviour: + +.. code-block:: python + + def test_parse_mutable_defaults_with_helper_macro_python(): + """A local Python equivalent of cParseArgsHelper.parse_mutable_defaults_with_helper_macro().""" + + def parse_mutable_defaults_with_helper_macro(obj, default_list=[]): + default_list.append(obj) + return default_list + + result = parse_mutable_defaults_with_helper_macro(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = parse_mutable_defaults_with_helper_macro(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = parse_mutable_defaults_with_helper_macro(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_macro(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_macro(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = parse_mutable_defaults_with_helper_macro(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 + +Now a similar test to establish the C behaviour: + +.. code-block:: python + + from cPyExtPatt import cParseArgsHelper + + def test_parse_mutable_defaults_with_helper_macro_c(): + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_macro(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_macro(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 diff --git a/doc/sphinx/source/pickle.rst b/doc/sphinx/source/pickle.rst new file mode 100644 index 0000000..92d63c6 --- /dev/null +++ b/doc/sphinx/source/pickle.rst @@ -0,0 +1,518 @@ +.. highlight:: c + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 2 + +.. index:: + single: Pickling + +==================================== +Pickling C Extension Types +==================================== + +If you need to provide support for pickling your specialised types from your C extension then you need to implement +some special functions. + +This example shows you how to provided pickle support for for the ``custom2.Custom`` type described in the C extension +tutorial in the +`Python documentation `_. +This defines an ``CustomObject`` object that has three fields; a first name, a last name and a number. +The ``CustomObject`` definition that needs to be pickled and un-pickled looks like this in C: + +.. code-block:: c + + typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; + } CustomObject; + +- The example C code is in ``src/cpy/Pickle/cCustomPickle.c``. +- The test code is in ``tests/unit/test_c_custom_pickle.py``. + +.. index:: + single: Pickling; Version Control + +Pickle Version Control +------------------------------- + +Since the whole point of ``pickle`` is persistence then pickled objects can hang around in databases, file systems, +data from the `shelve `_ module and whatnot for a long +time. +It is entirely possible that when un-pickled, sometime in the future, that your C extension has moved on and then +things become awkward. + +It is *strongly* recommended that you add some form of version control to your pickled objects. +In this example I just have a single integer version number which I write to the pickled object. +If the number does not match on unpickling then I raise an exception. +When I change the type API I would, judiciously, change this version number. + +Clearly more sophisticated strategies are possible by supporting older versions of the pickled object in some way but +this will do for now. + +We add some simple pickle version information to the C extension: + +.. code-block:: c + + + static const char* PICKLE_VERSION_KEY = "_pickle_version"; + static int PICKLE_VERSION = 1; + +Now we can implement ``__getstate__`` and ``__setstate__``, think of these as symmetric operations. + +First ``__getstate__``. + +.. index:: + single: Pickling; __getstate__ + +Implementing ``__getstate__`` +--------------------------------- + +``__getstate__`` pickles the object. +``__getstate__`` is expected to return a dictionary of the internal state of the ``Custom`` object. +Note that a ``Custom`` object has two Python objects (``first`` and ``last``) and a C integer (``number``) that need to +be converted to a Python object. +We also need to add the version information. + +Here is the C implementation: + +.. code-block:: c + + /* Pickle the object */ + static PyObject * + Custom___getstate__(CustomObject *self, PyObject *Py_UNUSED(ignored)) { + PyObject *ret = Py_BuildValue("{sOsOsisi}", + "first", self->first, + "last", self->last, + "number", self->number, + PICKLE_VERSION_KEY, PICKLE_VERSION); + return ret; + } + +.. note:: + + Note the careful use of ``Py_BuildValue()``. + ``"s"`` and ``"i"`` causes a new PyObject to be created in the dict. + ``"O"`` increments the reference count of an existing PyObject which is inserted into the dict. + + See :ref:`chapter_refcount.stolen.warning_pydict_setitem` + +.. index:: + single: Pickling; __setstate__ + +Implementing ``__setstate__`` +--------------------------------- + +The implementation of ``__setstate__`` un-pickles the object. +This is a little more complicated as there is quite a lot of error checking going on. +We are being passed an arbitrary Python object and need to check: + +* It is a Python dictionary. +* It has a version key and the version value is one that we can deal with. +* It has the required keys and values to populate our ``Custom`` object. + +Note that our ``__new__`` method (``Custom_new()``) has already been called on ``self``. +Before setting any member value we need to de-allocate the existing value set by ``Custom_new()`` otherwise we will have a memory leak. + +.. index:: + single: Pickling; __setstate__ + single: Pickling; __setstate__ Error Checking + +Error Checking +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: c + + /* Un-pickle the object */ + static PyObject * + Custom___setstate__(CustomObject *self, PyObject *state) { + /* Error check. */ + if (!PyDict_CheckExact(state)) { + PyErr_SetString(PyExc_ValueError, "Pickled object is not a dict."); + return NULL; + } + /* Version check. */ + /* Borrowed reference but no need to increment as we create a C long + * from it. */ + PyObject *temp = PyDict_GetItemString(state, PICKLE_VERSION_KEY); + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_Format(PyExc_KeyError, "No \"%s\" in pickled dict.", + PICKLE_VERSION_KEY); + return NULL; + } + if (!PyLong_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict version key \"%s\" is not a long but type \"%s\".", + PICKLE_VERSION_KEY, + temp->ob_type->tp_name + ); + return NULL; + } + int pickle_version = (int) PyLong_AsLong(temp); + if (pickle_version != PICKLE_VERSION) { + PyErr_Format(PyExc_ValueError, + "Pickle version mismatch. Got version %d but expected version %d.", + pickle_version, PICKLE_VERSION); + return NULL; + } + +Set the ``first`` Member +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: c + + /* NOTE: Custom_new() will have been invoked so self->first and self->last + * will have been allocated so we have to de-allocate them. */ + temp = PyDict_GetItemString(state, "first"); /* Borrowed reference. */ + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"first\" in pickled dict."); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"first\" is not a str but type \"%s\".", + temp->ob_type->tp_name + ); + return NULL; + } + Py_DECREF(self->first); + self->first = temp; + Py_INCREF(self->first); + +Set the ``last`` Member +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This code is very similar to the code for the first member above. + +.. code-block:: c + + temp = PyDict_GetItemString(state, "last"); /* Borrowed reference. */ + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"last\" in pickled dict."); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"last\" is not a str but type \"%s\".", + temp->ob_type->tp_name + ); + return NULL; + } + Py_DECREF(self->last); + self->last = temp; + Py_INCREF(self->last); + +Set the ``number`` Member +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is a C fundamental type so the code is slightly different: + +.. code-block:: c + + /* Borrowed reference but no need to incref as we create a C long from it. */ + PyObject *number = PyDict_GetItemString(state, "number"); + if (number == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"number\" in pickled dict."); + return NULL; + } + if (!PyLong_Check(number)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"number\" is not an int but type \"%s\".", + number->ob_type->tp_name + ); + return NULL; + } + self->number = (int) PyLong_AsLong(number); + +And we are done. + +.. code-block:: c + + Py_RETURN_NONE; + } + +``__setstate__`` in Full +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: c + + static PyObject * + Custom___setstate__(CustomObject *self, PyObject *state) { + + if (!PyDict_CheckExact(state)) { + PyErr_SetString(PyExc_ValueError, "Pickled object is not a dict."); + return NULL; + } + /* Version check. */ + /* Borrowed reference but no need to increment as we create a C long from it. */ + PyObject *temp = PyDict_GetItemString(state, PICKLE_VERSION_KEY); + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_Format(PyExc_KeyError, "No \"%s\" in pickled dict.", PICKLE_VERSION_KEY); + return NULL; + } + if (!PyLong_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict version key \"%s\" is not a long but type \"%s\".", + PICKLE_VERSION_KEY, + temp->ob_type->tp_name + ); + return NULL; + } + int pickle_version = (int) PyLong_AsLong(temp); + if (pickle_version != PICKLE_VERSION) { + PyErr_Format( + PyExc_ValueError, + "Pickle version mismatch. Got version %d but expected version %d.", + pickle_version, PICKLE_VERSION + ); + return NULL; + } + + temp = PyDict_GetItemString(state, "first"); /* Borrowed reference. */ + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"first\" in pickled dict."); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"first\" is not a str but type \"%s\".", + temp->ob_type->tp_name + ); + return NULL; + } + Py_DECREF(self->first); + self->first = temp; + Py_INCREF(self->first); + + temp = PyDict_GetItemString(state, "last"); /* Borrowed reference. */ + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"last\" in pickled dict."); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"last\" is not a str but type \"%s\".", + temp->ob_type->tp_name + ); + return NULL; + } + Py_DECREF(self->last); + self->last = temp; + Py_INCREF(self->last); + + /* Borrowed reference but no need to increment as we create a C long from it. */ + PyObject *number = PyDict_GetItemString(state, "number"); + if (number == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"number\" in pickled dict."); + return NULL; + } + if (!PyLong_Check(number)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"number\" is not an int but type \"%s\".", + number->ob_type->tp_name + ); + return NULL; + } + self->number = (int) PyLong_AsLong(number); + + Py_RETURN_NONE; + } + + +Add the Special Methods +--------------------------------- + +Now we need to add these two special methods to the methods table which now looks like this: + +.. code-block:: c + + static PyMethodDef Custom_methods[] = { + /* Existing methods here... */ + {"__getstate__", (PyCFunction) Custom___getstate__, METH_NOARGS, + "Pickle the Custom object" + }, + {"__setstate__", (PyCFunction) Custom___setstate__, METH_O, + "Un-pickle the Custom object" + }, + {NULL} /* Sentinel */ + }; + +Pickling a ``custom2.Custom`` Object +------------------------------------- + +We can test this with code like this that pickles one ``custom2.Custom`` object then creates another ``custom2.Custom`` +object from that pickle. +Here is some Python code that exercises our module (tests are in ``tests/unit/test_c_custom_pickle.py``): + +.. code-block:: python + + import io + import pickle + import pickletools + import sys + + import pytest + + from cPyExtPatt import cPickle + + + def test_module_dir(): + assert dir(cPickle) == [ + 'Custom', '__doc__', '__file__', '__loader__', + '__name__', '__package__', '__spec__', + ] + + + ARGS_FOR_CUSTOM_CLASS = ('FIRST', 'LAST', 11) + PICKLE_BYTES_FOR_CUSTOM_CLASS = ( + b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x12cPyExtPatt.cPickle\x94' + b'\x8c\x06Custom\x94\x93\x94)\x81\x94}\x94(\x8c\x05first\x94\x8c\x05FIRST' + b'\x94\x8c\x04last\x94\x8c\x04LAST\x94\x8c\x06number\x94K\x0b\x8c\x0f_pickle_' + b'version\x94K\x01ub.' + ) + + def test_pickle_getstate(): + custom = cPickle.Custom(*ARGS_FOR_CUSTOM_CLASS) + pickled_value = pickle.dumps(custom) + print() + print(f'Pickled original is {pickled_value}') + assert pickled_value == PICKLE_BYTES_FOR_CUSTOM_CLASS + # result = pickle.loads(pickled_value) + + + def test_pickle_setstate(): + custom = pickle.loads(PICKLE_BYTES_FOR_CUSTOM_CLASS) + assert custom.first == 'FIRST' + assert custom.last == 'LAST' + assert custom.number == 11 + + def test_pickle_round_trip(): + custom = cPickle.Custom(*ARGS_FOR_CUSTOM_CLASS) + pickled_value = pickle.dumps(custom) + result = pickle.loads(pickled_value) + assert id(result) != id(custom) + +So we have pickled one object and recreated a different, but equivalent, instance from the pickle of the original +object which is what we set out to do. + +.. index:: + single: Pickling; pickletools + +The Pickled Object in Detail +------------------------------------- + +If you are curious about the contents of the pickled object the the Python standard library provides the +`pickletools `_ module. +This allows you to inspect the pickled object. +Here is a test for that: + +.. code-block:: python + + import io + import pickle + import pickletools + import sys + + import pytest + + from cPyExtPatt import cPickle + + def test_pickletools(): + outfile = io.StringIO() + pickletools.dis(PICKLE_BYTES_FOR_CUSTOM_CLASS, out=outfile, annotate=1) + result = outfile.getvalue() + expected = '' + assert result == expected + +The expected output will be something like this: + +.. raw:: latex + + \begin{landscape} + +.. code-block:: text + + Pickled original is b'\x80\x04\x95[\x00\x00\x00\x00\x00\x00\x00\x8c\x07custom2\x94\x8c\x06Custom\x94\x93\x94)\x81\x94}\x94(\x8c\x05first\x94\x8c\x05FIRST\x94\x8c\x04last\x94\x8c\x04LAST\x94\x8c\x06number\x94K\x0b\x8c\x0f_pickle_version\x94K\x01ub.' + 0: \x80 PROTO 4 Protocol version indicator. + 2: \x95 FRAME 91 Indicate the beginning of a new frame. + 11: \x8c SHORT_BINUNICODE 'custom2' Push a Python Unicode string object. + 20: \x94 MEMOIZE (as 0) Store the stack top into the memo. The stack is not popped. + 21: \x8c SHORT_BINUNICODE 'Custom' Push a Python Unicode string object. + 29: \x94 MEMOIZE (as 1) Store the stack top into the memo. The stack is not popped. + 30: \x93 STACK_GLOBAL Push a global object (module.attr) on the stack. + 31: \x94 MEMOIZE (as 2) Store the stack top into the memo. The stack is not popped. + 32: ) EMPTY_TUPLE Push an empty tuple. + 33: \x81 NEWOBJ Build an object instance. + 34: \x94 MEMOIZE (as 3) Store the stack top into the memo. The stack is not popped. + 35: } EMPTY_DICT Push an empty dict. + 36: \x94 MEMOIZE (as 4) Store the stack top into the memo. The stack is not popped. + 37: ( MARK Push markobject onto the stack. + 38: \x8c SHORT_BINUNICODE 'first' Push a Python Unicode string object. + 45: \x94 MEMOIZE (as 5) Store the stack top into the memo. The stack is not popped. + 46: \x8c SHORT_BINUNICODE 'FIRST' Push a Python Unicode string object. + 53: \x94 MEMOIZE (as 6) Store the stack top into the memo. The stack is not popped. + 54: \x8c SHORT_BINUNICODE 'last' Push a Python Unicode string object. + 60: \x94 MEMOIZE (as 7) Store the stack top into the memo. The stack is not popped. + 61: \x8c SHORT_BINUNICODE 'LAST' Push a Python Unicode string object. + 67: \x94 MEMOIZE (as 8) Store the stack top into the memo. The stack is not popped. + 68: \x8c SHORT_BINUNICODE 'number' Push a Python Unicode string object. + 76: \x94 MEMOIZE (as 9) Store the stack top into the memo. The stack is not popped. + 77: K BININT1 11 Push a one-byte unsigned integer. + 79: \x8c SHORT_BINUNICODE '_pickle_version' Push a Python Unicode string object. + 96: \x94 MEMOIZE (as 10) Store the stack top into the memo. The stack is not popped. + 97: K BININT1 1 Push a one-byte unsigned integer. + 99: u SETITEMS (MARK at 37) Add an arbitrary number of key+value pairs to an existing dict. + 100: b BUILD Finish building an object, via __setstate__ or dict update. + 101: . STOP Stop the unpickling machine. + highest protocol among opcodes = 4 + +.. raw:: latex + + \end{landscape} + +.. index:: + single: Pickling; External State + +Pickling Objects with External State +----------------------------------------- + +This is just a simple example, if your object relies on external state such as open files, databases and the like you +need to be careful, and knowledgeable about your state management. +There is some useful information in +`Handling Stateful Objects `_ + + +.. note:: + + Marshalling support for use with the `marshall `_ + module is given by the `C Marshall API `_ + +.. index:: + single: Pickling; References + +References +----------------------- + +* Python API documentation for `__setstate__ `_ +* Python API documentation for `__getstate__ `_ +* Useful documentation for `Handling Stateful Objects `_ +* Python `pickle module `_ +* Python `shelve module `_ diff --git a/doc/sphinx/source/refcount.rst b/doc/sphinx/source/refcount.rst index 8bf5d61..37cf2af 100644 --- a/doc/sphinx/source/refcount.rst +++ b/doc/sphinx/source/refcount.rst @@ -4,21 +4,29 @@ .. toctree:: :maxdepth: 3 -============ -Introduction -============ +.. + .. _Reference Counting: https://docs.python.org/3.9/c-api/refcounting.html -Writing Python C Extensions can be daunting; you have to cast aside the security and fluidity of Python and embrace C, not just C but Pythons C API, which is huge [#]_. Not only do you have to worry about just your standard ``malloc()`` and ``free()`` cases but now you have to contend with how CPython's does its memory management which is by *reference counting*. +.. _Py_REFCNT(): https://docs.python.org/3.9/c-api/structures.html#c.Py_REFCNT +.. _Py_INCREF(): https://docs.python.org/3.9/c-api/refcounting.html#c.Py_INCREF +.. _Py_XINCREF(): https://docs.python.org/3.9/c-api/refcounting.html#c.Py_XINCREF +.. _Py_DECREF(): https://docs.python.org/3.9/c-api/refcounting.html#c.Py_DECREF +.. _Py_XDECREF(): https://docs.python.org/3.9/c-api/refcounting.html#c.Py_XDECREF -I describe some of the pitfalls you (I am thinking of you as a savvy C coder) can encounter and some of the coding patterns that you can use to avoid them. +.. _chapter_refcount: -First up: understanding reference counts and Python's terminology. +.. index:: + single: Reference Counts + single: Reference Counts; New + single: Reference Counts; Stolen + single: Reference Counts; Borrowed ================================= PyObjects and Reference Counting ================================= -A ``PyObject`` can represent any Python object. It is a fairly minimal C struct consisting of a reference count and a pointer to the object proper: +A ``PyObject`` can represent any Python object. It is a fairly minimal C struct consisting of a reference count and a +pointer to the object type [#]_: .. code-block:: c @@ -27,9 +35,49 @@ A ``PyObject`` can represent any Python object. It is a fairly minimal C struct struct _typeobject *ob_type; } PyObject; -In Python C extensions you always create and deallocate these ``PyObjects`` *indirectly*. Creation is via Python's C API and destruction is done by decrementing the reference count. If this count hits zero then CPython will free all the resources used by the object. +.. note:: -Here is an example of a normal ``PyObject`` creation and deallocation: + The ``struct _typeobject`` is crucial for defining new types. + This structure changes over various Python versions. + For convenience I have included the structure definitions for Python types in ``type_objects/`` for Python versions + 3.6 to 3.13 which allows easy comparison between versions. + +In Python C extensions you always create and deallocate these ``PyObjects`` *indirectly*. +Creation is via Python's C API and destruction is done by decrementing the reference count. +If this count hits zero then CPython will free all the resources used by the object. + +The macros to manipulate reference counts are: + +.. list-table:: Reference Count Macros + :widths: 20 85 + :header-rows: 1 + + * - Macro + - Description + * - `Py_REFCNT()`_ + - Get the reference count of an object. + This expands to ``(((PyObject*)(o))->ob_refcnt)``. + This will segfault if passed ``NULL``. + * - `Py_INCREF()`_ + - Increments the reference count of the given ``PyObject *``. + This will segfault if passed ``NULL``. + * - `Py_XINCREF()`_ + - As `Py_INCREF()`_ but does nothing if passed ``NULL``. + * - `Py_DECREF()`_ + - Decrements the reference count of the given ``PyObject *``. + This will segfault if passed ``NULL``. + If the reference count reaches zero then the object will be deallocated and the memory *may* be reused. + + **Warning:** De-allocation might cause the execution of arbitrary Python code has free access to all Python + global variables. + + **Warning:** Calling this when the reference count is 1 might reuse the memory so on exit the object reference + count could be anything. + * - `Py_XDECREF()`_ + - As `Py_DECREF()`_ but does nothing if passed ``NULL``. + + +Here is an example of a normal ``PyObject`` creation, print and de-allocation: .. code-block:: c :linenos: @@ -40,23 +88,30 @@ Here is an example of a normal ``PyObject`` creation and deallocation: PyObject *pObj = NULL; pObj = PyBytes_FromString("Hello world\n"); /* Object creation, ref count = 1. */ - PyObject_Print(pLast, stdout, 0); + PyObject_Print(pObj, stdout, 0); Py_DECREF(pObj); /* ref count becomes 0, object deallocated. * Miss this step and you have a memory leak. */ } The twin challenges in Python extensions are: -* Avoiding undefined behaviour such as object access after an object's reference count is zero. This is analogous in C to access after ``free()`` or a using `dangling pointer `_. -* Avoiding memory leaks where an object's reference count never reaches zero and there are no references to the object. This is analogous in C to a ``malloc()`` with no corresponding ``free()``. +* Avoiding undefined behaviour such as object access after an object's reference count is zero. + This is analogous in C to access after ``free()`` or a using + `dangling pointer `_. +* Avoiding memory leaks where an object's reference count never reaches zero and there are no references to the object. + This is analogous in C to a ``malloc()`` with no corresponding ``free()``. Here are some examples of where things can go wrong: +.. index:: + single: Access After Free + ----------------------- Access After Free ----------------------- -Taking the above example of a normal ``PyObject`` creation and deallocation then in the grand tradition of C memory management after the ``Py_DECREF`` the ``pObj`` is now referencing free'd memory: +Taking the above example of a normal ``PyObject`` creation and deallocation then in the grand tradition of C memory +management after the ``Py_DECREF`` the ``pObj`` is now referencing free'd memory: .. code-block:: c :linenos: @@ -64,38 +119,45 @@ Taking the above example of a normal ``PyObject`` creation and deallocation then #include "Python.h" void print_hello_world(void) { - PyObject *pObj = NULL: + PyObject *pObj = NULL; pObj = PyBytes_FromString("Hello world\n"); /* Object creation, ref count = 1. */ - PyObject_Print(pLast, stdout, 0); + PyObject_Print(pObj, stdout, 0); Py_DECREF(pObj); /* ref count = 0 so object deallocated. */ /* Accidentally use pObj... */ + PyObject_Print(pObj, stdout, 0); } Accessing ``pObj`` may or may not give you something that looks like the original object. -The corresponding issue is if you decrement the reference count without previously incrementing it then the caller might find *their* reference invalid: +The corresponding issue is if you decrement the reference count without previously incrementing it then the caller +might find *their* reference invalid: .. code-block:: c :linenos: static PyObject *bad_incref(PyObject *pObj) { - /* Forgotten Py_INCREF(pObj); here... */ - + /* Use pObj... */ Py_DECREF(pObj); /* Might make reference count zero. */ Py_RETURN_NONE; /* On return caller might find their object free'd. */ } -After the function returns the caller *might* find the object they naively trusted you with but probably not. A classic access-after-free error. +After the function returns the caller *might* find the object they naively trusted you with but probably not. +A classic access-after-free error. + +.. index:: + single: Memory Leaks ----------------------- Memory Leaks ----------------------- -Memory leaks occur with a ``PyObject`` if the reference count never reaches zero and there is no Python reference or C pointer to the object in scope. -Here is where it can go wrong: in the middle of a great long function there is an early return on error. On that path this code has a memory leak: +Memory leaks occur with a ``PyObject`` if the reference count never reaches zero and there is no Python reference or C +pointer to the object in scope. +Here is where it can go wrong: in the middle of a great long function there is an early return on error. +On that path this code has a memory leak: .. code-block:: c :linenos: @@ -112,7 +174,8 @@ Here is where it can go wrong: in the middle of a great long function there is a Py_RETURN_NONE; } -The problem is that the reference count was not decremented before the early return, if ``pObj`` was a 100 Mb string then that memory is lost. Here is some C code that demonstrates this: +The problem is that the reference count was not decremented before the early return, if ``pObj`` was a 100 Mb string +then that memory is lost. Here is some C code that demonstrates this: .. code-block:: c @@ -121,13 +184,14 @@ The problem is that the reference count was not decremented before the early ret Py_RETURN_NONE; } -And here is what happens to the memory if we use this function from Python (``cPyRefs.incref(...)`` in Python calls ``bad_incref()`` in C):: +And here is what happens to the memory if we use this function from Python (``cPyRefs.incref(...)`` in Python calls +``bad_incref()`` in C):: - >>> import cPyRefs # Process uses about 1Mb + >>> from cPyExtPatt import cPyRefs # Process uses about 1Mb >>> s = ' ' * 100 * 1024**2 # Process uses about 101Mb >>> del s # Process uses about 1Mb >>> s = ' ' * 100 * 1024**2 # Process uses about 101Mb - >>> cPyRefs.incref(s) # Now do an increment without decrement + >>> cPyRefs.bad_incref(s) # Now do an increment without decrement >>> del s # Process still uses about 101Mb - leaked >>> s # Officially 's' does not exist Traceback (most recent call last): @@ -135,14 +199,68 @@ And here is what happens to the memory if we use this function from Python (``cP NameError: name 's' is not defined >>> # But process still uses about 101Mb - 's' is leaked + + +.. _chapter_refcount.warning_ref_count_unity: + +------------------------------------------------------- +Warning on Relying on Reference Counts of Unity or Less +------------------------------------------------------- + .. warning:: - Do not be tempted to read the reference count itself to determine if the object is alive. The reason is that if ``Py_DECREF`` sees a refcount of one it can free and then reuse the address of the refcount field for a completely different object which makes it highly unlikely that that field will have a zero in it. There are some examples of this later on. - + Do not be tempted to read the reference count itself to determine if the object is alive. + The reason is that if ``Py_DECREF`` sees a refcount of one it can free and then reuse the address of the refcount + field for a completely different object which makes it highly unlikely that that field will have a zero in it. + There are some examples of this later on. + + Here is a simple example: + + .. code-block:: c + + PyObject *op = PyUnicode_FromString("My test string."); + assert(Py_REFCNT(op) == 1); + Py_DECREF(op); + /* Py_REFCNT(op) can be anything here. */ + + For example this code is asking for trouble: + + .. code-block:: c + + PyObject *op; + /* Do something ... */ + while (op->ob_refcnt) { + Py_DECREF(op); + } + + This will either loop forever or segfault. + +.. _chapter_refcount.pythons_garbage_collector: + +------------------------------------------------------- +Python's Garbage Collector +------------------------------------------------------- + +CPython is a garbage collected language however the primary means of garbage collection is by reference counting. +The 'garbage collector' is a secondary device that attempts to resolve circular references. +If the reference counts are wrong the 'garbage collector' can do nothing to recover memory [#]_. + +For example this will leak whatever efforts CPython's garbage collector makes: + +.. code-block:: c + + void leak(void) { + PyObject *value = PyLong_FromLong(123456L); + /* value has been allocated on the heap with a reference count of 1. */ + /* On return, value will be leaked. */ + } + ----------------------- Python Terminology ----------------------- -The Python documentation uses the terminology "New", "Stolen" and "Borrowed" references throughout. These terms identify who is the *real owner* of the reference and whose job it is to clean it up when it is no longer needed: + +The Python documentation uses the terminology "New", "Stolen" and "Borrowed" references throughout. +These terms identify who is the *real owner* of the reference and whose job it is to clean it up when it is no longer needed: * **New** references occur when a ``PyObject`` is constructed, for example when creating a new list. * **Stolen** references occur when composing a ``PyObject``, for example appending a value to a list. "Setters" in other words. @@ -152,6 +270,11 @@ This is about programming by contract and the following sections describe the co First up **New** references. +.. index:: + single: Reference Counts; New + +.. _chapter_refcount.new: + ^^^^^^^^^^^^^^^^^^ "New" References ^^^^^^^^^^^^^^^^^^ @@ -160,9 +283,10 @@ When you create a "New" ``PyObject`` from a Python C API then you own it and it * Dispose of the object when it is no longer needed with ``Py_DECREF`` [#]_. * Give it to someone else who will do that for you. -If neither of these things is done you have a memory leak in just like a ``malloc()`` without a corresponding ``free()``. +If neither of these things is done you have a memory leak just like a ``malloc()`` without a corresponding ``free()``. -Here is an example of a well behaved C function that take two C longs, converts them to Python integers and, subtracts one from the other and returns the Python result: +Here is an example of a well behaved C function that take two C longs, converts them to Python integers and, subtracts +one from the other and returns the Python result: .. code-block:: c :linenos: @@ -180,25 +304,91 @@ Here is an example of a well behaved C function that take two C longs, converts ``PyLong_FromLong()`` returns a *new* reference which means we have to clean up ourselves by using ``Py_DECREF``. -``PyNumber_Subtract()`` also returns a *new* reference but we expect the caller to clean that up. If the caller doesn't then there is a memory leak. +``PyNumber_Subtract()`` also returns a *new* reference but we expect the caller to clean that up. +If the caller doesn't then there is a memory leak. -So far, so good but what would be really bad is this: +So far, so good but what would be bad is this: .. code-block:: c r = PyNumber_Subtract(PyLong_FromLong(a), PyLong_FromLong(b)); -You have passed in two *new* references to ``PyNumber_Subtract()`` and that function has no idea that they have to be decref'd once used so the two PyLong objects are leaked. +You have passed in two *new* references to ``PyNumber_Subtract()`` and that function has no idea that they have to be +decref'd once used so the two PyLong objects are leaked. -The contract with *new* references is: either you decref it or give it to someone who will. If neither happens then you have a memory leak. +The contract with *new* references is: either you decref it or give it to someone who will. +If neither happens then you have a memory leak. + +An Example Leak Problem +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here is an example that exhibits a leak. The object is to add the integers 400 to 404 to the end of a list. You might want to study it to see if you can spot the problem: + +.. code-block:: c + + static PyObject * + list_append_one_to_four(PyObject *list) { + for (int i = 400; i < 405; ++i) { + PyList_Append(list, PyLong_FromLong(i)); + } + Py_RETURN_NONE; + } + +The problem is that ``PyLong_FromLong`` creates ``PyObject`` (an int) with a reference count of 1 **but** ``PyList_Append`` increments the reference count of the object passed to it by 1 to 2. This means when the list is destroyed the list element reference counts drop by one (to 1) but *no lower* as nothing else references them. Therefore they never get deallocated so there is a memory leak. + + +The append operation *must* behave this way, consider this Python code + +.. code-block:: python + + l = [] + a = 400 + # The integer object '400' has a reference count of 1 as only + # one symbol references it: a. + l.append(a) + # The integer object '400' must now have a reference count of + # 2 as two symbols reference it: a and l, specifically l[-1]. + +The fix is to create a temporary item and then decref *that* once appended (error checking omitted): + +.. code-block:: c + + static PyObject * + list_append_one_to_four(PyObject *list) { + PyObject *temporary_item = NULL; + + for (int i = 400; i < 405; ++i) { + /* Create the object to append to the list. */ + temporary_item = PyLong_FromLong(i); + /* temporary_item->ob_refcnt == 1 now */ + /* Append it. This will increment the reference count to 2. */ + PyList_Append(list, temporary_item); + /* Decrement our reference to it leaving the list having the only reference. */ + Py_DECREF(temporary_item); + /* temporary_item->ob_refcnt == 1 now */ + temporary_item = NULL; /* Good practice... */ + } + Py_RETURN_NONE; + } + +.. index:: + single: Reference Counts; Stolen + +.. _chapter_refcount.stolen: ^^^^^^^^^^^^^^^^^^^^^^^ "Stolen" References ^^^^^^^^^^^^^^^^^^^^^^^ -This is also to do with object creation but where another object takes responsibility for decref'ing (possibly freeing) the object. Typical examples are when you create a ``PyObject`` that is then inserted into an existing container such as a tuple list, dict etc. +This is also to do with object creation but where another object takes responsibility for decref'ing (possibly freeing) +the object. +Typical examples are when you create a ``PyObject`` that is then inserted into an existing container such as a tuple, +list, dict etc. -The analogy with C code is malloc'ing some memory, populating it and then passing that pointer to a linked list which then takes on the responsibility to free the memory if that item in the list is removed. If you were to free the memory you had malloc'd then you will get a double free when the linked list (eventually) frees its members. +The analogy with C code is malloc'ing some memory, populating it and then passing that pointer to a linked list which +then takes on the responsibility to free the memory if that item in the list is removed. +If you were to free the memory you had malloc'd then you will get a double free when the linked list (eventually) +frees its members. Here is an example of creating a 3-tuple, the comments describe what is happening contractually: @@ -222,7 +412,10 @@ Here is an example of creating a 3-tuple, the comments describe what is happenin return r; /* Callers responsibility to decref. */ } -Note line 10 where we are overwriting an existing pointer with a new value, this is fine as ``r`` has taken responsibility for the first pointer value. This pattern is somewhat alarming to dedicated C programmers so the more common pattern, without the assignment to ``v`` is shown in line 13. +Note line 10 where we are overwriting an existing pointer with a new value, this is fine as ``r`` has taken +responsibility for the original pointer value. +This pattern is somewhat alarming to dedicated C programmers so the more common pattern, without the assignment to +``v`` is shown in line 13. What would be bad is this: @@ -232,25 +425,75 @@ What would be bad is this: PyTuple_SetItem(r, 0, v); /* r takes ownership of the reference. */ Py_DECREF(v); /* Now we are interfering with r's internals. */ -Once ``v`` has been passed to ``PyTuple_SetItem`` then your ``v`` becomes a *borrowed* reference with all of their problems which is the subject of the next section. +Once ``v`` has been passed to ``PyTuple_SetItem`` then your ``v`` becomes a *borrowed* reference with all of their +problems which is the subject of the next section. + +The contract with *stolen* references is: the thief will take care of things so you don't have to. +If you try to the results are undefined. + +.. _chapter_refcount.stolen.warning_pydict_setitem: + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Warning on "Stolen" References With Containers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The contract with *stolen* references is: the thief will take care of things so you don't have to. If you try to the results are undefined. +.. warning:: + + The example above describes tuples that *do* "steal" references. + Other containers, such as a ``dict`` and ``set`` do *not*. + + A consequence is that ``PyTuple_SetItem(pTuple, 0, PyLong_FromLong(12345L))`` does *not* leak + with *new* references but ``PyDict_SetItem(pDict, PyLong_FromLong(12345L), PyLong_FromLong(123456L))`` + *does* leak with *new* references. + To avoid that particular leak then create temporaries, then call ``PyDict_SetItem`` with them and then decref + your temporaries. + If the key/value objects are *borrowed* references then there is nothing to do, ``PyDict_SetItem`` + will increment them correctly. + + Unfortunately this was only made clear in the Python documentation for ``PyDict_SetItem`` in Python version 3.8+: + https://docs.python.org/3.8/c-api/dict.html + + This also happens with `Py_BuildValue `_ when building + with newly created Python objects (using ``Py_BuildValue("{OO}", ...``). + + This warning also applies to `PySet_Add() `_ which also + increments the reference rather than stealing it. + The documentation does not mention this at all (as of Python 3.13). + + See ``src/cpy/RefCount/cRefCount.c`` and ``tests/unit/test_c_ref_count.py`` for verification of this. + + The next chapter :ref:`chapter_containers_and_refcounts` goes into more detail about this, how it can bite you + and how you can defend yourself from these issues. + +.. index:: + single: Reference Counts; Borrowed + +.. _chapter_refcount.borrowed: ^^^^^^^^^^^^^^^^^^^^^^^ -"Borrowed" References +"Borrowed" References ^^^^^^^^^^^^^^^^^^^^^^^ -When you obtain a reference to an existing ``PyObject`` in a container using a 'getter' you are given a *borrowed* reference and this is where things can get tricky. The most subtle bugs in Python C Extensions are usually because of the misuse of borrowed references. +When you obtain a reference to an existing ``PyObject`` in a container using a 'getter' you are given a *borrowed* +reference and this is where things can get tricky. +The most subtle bugs in Python C Extensions are usually because of the misuse of borrowed references. + +If you find the term "borrowed references" mystifying then perhaps "shared references" might be clearer, because that +is exactly what they are. -The analogy in C is having two pointers to the same memory location: so who is responsible for freeing the memory and what happens if the other pointer tries to access that free'd memory? +The analogy in C is having two pointers to the same memory location: so who is responsible for freeing the memory and +what happens if the other pointer tries to access that free'd memory? -Here is an example where we are accessing the last member of a list with a "borrowed" reference. This is the sequence of operations: +Here is an example where we are accessing the last member of a list with a "borrowed" reference. +This is the sequence of operations: * Get a *borrowed* reference to a member of the list. * Do some operation on that list, in this case call ``do_something()``. * Access the *borrowed* reference to the member of the original list, in this case just print it out. -Here is a C function that *borrows* a reference to the last object in a list, prints out the object's reference count, calls another C function ``do_something()`` with that list, prints out the reference count of the object again and finally prints out the Python representation of the object: +Here is a C function that *borrows* a reference to the last object in a list, prints out the object's reference count, +calls another C function ``do_something()`` with that list, prints out the reference count of the object again and +finally prints out the Python representation of the object: .. code-block:: c :linenos: @@ -270,87 +513,131 @@ Here is a C function that *borrows* a reference to the last object in a list, pr The problem is that if ``do_something()`` mutates the list it might invalidate the item that we have a pointer to. -Suppose ``do_something()`` 'removes' every item in the list [#]_. Then whether reference ``pLast`` is still "valid" depends on what other references to it exist and you have no control over that. Here are some examples of what might go wrong in that case (C ``pop_and_print_BAD`` is mapped to the Python ``cPyRefs.popBAD``):: +Suppose ``do_something()`` 'removes' every item in the list [#]_. +Such as: + +.. code-block:: c + + void do_something(PyObject *pList) { + while (PyList_Size(pList) > 0) { + PySequence_DelItem(pList, 0); + } + } + +Then whether reference ``pLast`` is still "valid" depends on what other references to it exist and you have no control +over that. + +Here are some examples of what might go wrong in that case. + +.. code-block:: python >>> l = ["Hello", "World"] - >>> cPyRefs.popBAD(l) # l will become empty + >>> cPyRefs.pop_and_print_BAD(l) # l will become empty Ref count was: 1 Ref count now: 4302027608 'World' The reference count is bogus, however the memory has not been *completely* overwritten so the object (the string "World") *appears* to be OK. -If we try a different string:: +If we try a different string: + +.. code-block:: python >>> l = ['abc' * 200] - >>> cPyRefs.popBAD(l) + >>> cPyRefs.pop_and_print_BAD(l) Ref count was: 1 Ref count now: 2305843009213693952 Segmentation fault: 11 At least this will get your attention! -Incidentially from Python 3.3 onwards there is a module `faulthandler `_ that can give useful debugging information (file ``FaultHandlerExample.py``): +.. index:: + single: faulthandler -.. code-block:: python - :linenos: - :emphasize-lines: 5 +.. note:: + + Incidentally from Python 3.3 onwards there is a module + `faulthandler `_ + that can give useful debugging information (file ``FaultHandlerExample.py``): - import faulthandler - faulthandler.enable() - import cPyRefs - l = ['abc' * 200] - cPyRefs.popBAD(l) + .. code-block:: python + :linenos: + :emphasize-lines: 5 -And this is what you get: + import faulthandler + faulthandler.enable() + from cPyExtPatt import cPyRefs + l = ['abc' * 200] + cPyRefs.pop_and_print_BAD(l) -.. code-block:: console + And this is what you get: - $ python3 FaultHandlerExample.py - Ref count was: 1 - Ref count now: 2305843009213693952 - Fatal Python error: Segmentation fault + .. code-block:: console - Current thread 0x00007fff73c88310: - File "FaultHandlerExample.py", line 7 in - Segmentation fault: 11 + $ python3 FaultHandlerExample.py + Ref count was: 1 + Ref count now: 2305843009213693952 + Fatal Python error: Segmentation fault -There is a more subtle issue; suppose that in your Python code there is a reference to the last item in the list, then the problem suddenly "goes away":: + Current thread 0x00007fff73c88310: + File "FaultHandlerExample.py", line 7 in + Segmentation fault: 11 + +"Borrowed" References Go Bad +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There is a more subtle issue; suppose that in your Python code there is a reference to the last item in the list, +then the problem suddenly "goes away": + +.. code-block:: python >>> l = ["Hello", "World"] >>> a = l[-1] - >>> cPyRefs.popBAD(l) + >>> cPyRefs.pop_and_print_BAD(l) Ref count was: 2 Ref count now: 1 'World' -The reference count does not go to zero so the object is preserved. The problem is that the correct behaviour of your C function depends entirely on that caller code having a extra reference. +The reference count does not go to zero so the object is preserved. +The problem is that the correct behaviour of your C function depends entirely on that caller code having a extra +reference. + +This can happen implicitly as well: -This can happen implicitly as well:: +.. code-block:: python >>> l = list(range(8)) - >>> cPyRefs.popBAD(l) + >>> cPyRefs.pop_and_print_BAD(l) Ref count was: 20 Ref count now: 19 7 -The reason for this is that (for efficiency) CPython maintains the integers -5 to 255 permanently so they never go out of scope. If you use different integers we are back to the same access-after-free problem:: +The reason for this is that (for efficiency) CPython maintains the integers -5 to 255 permanently so they never go out +of scope. +If you use different integers we are back to the same access-after-free problem: + +.. code-block:: python >>> l = list(range(800,808)) - >>> cPyRefs.popBAD(l) + >>> cPyRefs.pop_and_print_BAD(l) Ref count was: 1 Ref count now: 4302021872 807 -The problem with detecting these errors is that the bug is data dependent so your code might run fine for a while but some change in user data could cause it to fail. And it will fail in a manner that is not easily detectable. +The problem with detecting these errors is that the bug is data dependent so your code might run fine for a while but +some change in user data could cause it to fail. And it will fail in a manner that is not easily detectable. -Fortunately the solution is easy: with borrowed references you should increment the reference count whilst you have an interest in the object, then decrement it when you no longer want to do anything with it: +"Borrowed" References Solution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Fortunately the solution is easy: with borrowed references you should increment the reference count whilst you have an +interest in the object, then decrement it when you no longer want to do anything with it: .. code-block:: c :linenos: :emphasize-lines: 5,7-8 - static PyObject *pop_and_print_BAD(PyObject *pList) { + static PyObject *pop_and_print_GOOD(PyObject *pList) { PyObject *pLast; pLast = PyList_GetItem(pList, PyList_Size(pList) - 1); @@ -364,34 +651,214 @@ Fortunately the solution is easy: with borrowed references you should increment Py_RETURN_NONE; } -The ``pLast = NULL;`` line is not necessary but is good coding style as it will cause any subsequent acesses to ``pLast`` to fail. +The ``pLast = NULL;`` line is not necessary but is good coding style as it will cause any subsequent accesses to +``pLast`` to fail. + +An important takeaway here is that incrementing and decrementing reference counts is a cheap operation but the +consequences of getting it wrong can be expensive. +A precautionary approach in your code might be to *always* increment borrowed references when they are instantiated +and then *always* decrement them before they go out of scope. +That way you incur two cheap operations but eliminate a vastly more expensive one. + + +.. index:: + single: Strong References + single: Weak References + single: References; Strong + single: References; Weak + +-------------------------- +Strong and Weak References +-------------------------- + +Another mental model to look at this is the concept of *strong* and *weak* references to an object. +This model is commonly used in other software domains [#]_. +If this model suits you then use it! + +Here are the essential details between this model and the Python one. +The mapping between the Python new/stolen/borrowed terminology and strong/weak terminology is: -An important takeaway here is that incrementing and decrementing reference counts is a cheap operation but the consequences of getting it wrong can be expensive. A precautionary approach in your code might be to *always* increment borrowed references when they are instantiated and then *always* decrement them before they go out of scope. That way you incur two cheap operations but eliminate a vastly more expensive one. +* A "new" reference is a single *strong* reference. +* A "stolen" reference is handing over responsibility for a *strong* reference, any previous reference becomes a *weak* + reference. +* A "borrowed" reference is a new *weak* reference. + +And the Python implementation gives: + +* ``Py_REFCNT()`` returns the number of *strong* references. +* ``Py_INCREF()`` creates a new *strong* reference. +* ``Py_DECREF()`` releases a *strong* reference. + +The two types of errors are: + +* Any unreachable *strong* reference is a memory leak (an unreachable object with a reference count > 0). +* Accessing an object through a *weak* reference where no *strong* references exist leads to undefined behaviour. + Note the warning above: :ref:`chapter_refcount.warning_ref_count_unity`. ^^^^^^^^^^^^^^^^^^ -Summary +Examples ^^^^^^^^^^^^^^^^^^ +Unreachable Strong Reference +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here we create a new object with a *strong* reference count of unity and then abandon it: + +.. code-block:: c + + void unreachable(void) { + /* Create a new object, a strong reference (a "new" reference). */ + PyObject *value = PyUnicode_FromString("My test string."); + /* Check the strong reference count. */ + assert(Py_REFCNT(value) == 1); + /* The object is now unreachable as 'value' goes out of scope. + * The strong reference count is still unity so the object is leaked. */ + return; + } + +"New" References +^^^^^^^^^^^^^^^^^^ + +Here is a walked through example of the lifetime of creating a tuple containing a single string. +First create the string: + +.. code-block:: c + + Py_ssize_t strong_ref_count; + + PyObject *value = PyUnicode_FromString("My test string."); + strong_ref_count = Py_REFCNT(value); + assert(strong_ref_count == 1); + +There is one strong reference to the string and it is 'owned' by ``value``. +Now create a tuple: + +.. code-block:: c + + PyObject *container = PyTuple_New(1); + strong_ref_count = Py_REFCNT(container); + assert(strong_ref_count == 1); + +There is one strong reference to the container and it is 'owned' by ``container``. +Now insert the value into the tuple (error checking omitted): + +.. code-block:: c + + PyTuple_SetItem(container, 0, value); + strong_ref_count = Py_REFCNT(value); + assert(strong_ref_count == 1); + +At this point, in the strong/weak model, we have two references to the original string. +One is *strong* (since ``Py_REFCNT(value)`` is 1), the other must then be a *weak* reference. +But which is which? +In other words is ``value`` or ``container[0]`` *strong* or *weak*? + +This model determines that ``container`` holds the *strong* reference since on destruction of that container +``Py_DECREF()`` will be called on that reference reducing the strong reference count to zero. +Therefore ``value``, once a *strong* reference is now a *weak* reference. +This expresses the concept of *stealing* a reference. +So we end up with: + +* A *strong* reference which is held by ``container``, specifically ``container[0]``. +* A *weak* reference which is held by ``value``. + +Now lets destroy the container. + +.. code-block:: c + + Py_DECREF(container); + +This will destroy the contents, by remove one *strong* reference for each value +of the container then removing the *strong* reference to the container. +This now makes ``container`` a weak reference to an object that has no *strong* references: + +So we are left with no strong references but still have two weak references, held by ``value`` and ``container``. +Now accessing an object that has no strong references through a weak reference is undefined behaviour such +as this: + +.. code-block:: c + + PyObject_Print(container, stdout, Py_PRINT_RAW); + PyObject_Print(value, stdout, Py_PRINT_RAW); + +Python's documentation on `strong references `_ +and `borrowed (weak) references `_. + +.. _chapter_refcount.a_possible_precautionary_principle: + +---------------------------------- +A Possible Precautionary Principle +---------------------------------- + +Getting the reference counts wrong at any stage of the program risks: + +* If the reference count is unnecessarily *high* you will have a memory leak. +* If the reference count is unnecessarily *low* you might have a segmentation fault or undefined behaviour. + +Note that memory leaks are usually harder, sometimes *much* harder, to fix than segmentation faults. + +To avoid both possibilities you need to get the reference counts *exactly* right and requires a great attention to +detail which is expensive to do. +This problem is exacerbated if the code base is large and constantly changing. + +So are there some other trade-offs that could be made? +Well if your process is short running you might not care about memory leaks at all as the OS will reclaim all the +memory at the process end. +In this case you could make the choice to ignore decrementing reference counts (or gratuitously increase the reference +count) so that objects are never free'd thus a segmentation fault is impossible. +This is similar to the way some compilers use ``malloc()`` but never bother with ``free()``. +The rationale is that there may be any number of references to an internal data structure and it is dangerous to +invalidate any of them whereas if the compiler is a short running process then leaks are unimportant (you hope). +This is also referred to as the +`Null Garbage Collector [devblogs.microsoft.com] `_. + +However if you want to create a Python Extension for a long running process (say a server) and you can't put up with +memory leaks then you have no choice but to control the reference counts carefully. + +Budget accordingly. + +----------------------- +Summary +----------------------- + The contracts you enter into with these three reference types are: -============== =============== -Type Contract -============== =============== -**New** Either you decref it or give it to someone who will, otherwise you have a memory leak. -**Stolen** The thief will take of things so you don't have to. If you try to the results are undefined. -**Borrowed** The lender can invalidate the reference at any time without telling you. Bad news. So increment a borrowed reference whilst you need it and decrement it when you are finished. -============== =============== +.. list-table:: Python References. + :widths: 15 85 + :header-rows: 1 + + * - Type + - Contract + * - **New** + - Either you decref it or give it to someone who will, otherwise you have a memory leak. + * - **Stolen** + - The thief will take of things so you don't have to. If you try to the results are undefined. + * - **Borrowed** + - The lender can invalidate the reference at any time without telling you. + Bad news. + So increment a borrowed reference whilst you need it and decrement it when you are finished. +The strong reference/weak reference model maps well to the Python model. + +In the next chapter :ref:`chapter_containers_and_refcounts` we look in more detail about the interplay of reference +counts with Python objects and Python containers such as ``tuple``, ``list``, ``set`` and ``dict``. .. rubric:: Footnotes -.. [#] Huge, but pretty consistent once mastered. -.. [#] To be picky we just need to decrement the use of *our* reference to it. Other code that has incremented the same reference is responsible for decrementing their use of the reference. -.. [#] Of course we never *remove* items in a list we merely decrement their reference count (and if that hits zero then they are deleted). Such as: -.. code-block:: python +.. [#] This varies slightly between different Python versions. - void do_something(PyObject *pList) { - while (PyList_Size(pList) > 0) { - PySequence_DelItem(pList, 0); - } - } +.. [#] If you are interested in some of the technical details of memory management terms and techniques in a wide + variety of languages then a great place to start is ``_. + For example here is their entry about + `reference counting `_. + +.. [#] To be picky we just need to decrement the use of *our* reference to it. + Other code that has incremented the same reference is responsible for decrementing their use of the reference. + +.. [#] Of course we never *remove* items in a list we merely decrement their reference count, + and if that hits zero then they are deleted. + +.. [#] Here is ``_'s definition of a + `strong reference `_ + and a + `weak reference `_. diff --git a/doc/sphinx/source/simple_example.rst b/doc/sphinx/source/simple_example.rst new file mode 100644 index 0000000..686255d --- /dev/null +++ b/doc/sphinx/source/simple_example.rst @@ -0,0 +1,273 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + +================= +A Simple Example +================= + +This very artificial example illustrates some of the benefits and drawbacks of Python C Extensions. + +Suppose you have some Python code such as this that is performing slowly: + +.. code-block:: python + + def fibonacci(index: int) -> int: + if index < 2: + return index + return fibonacci(index - 2) + fibonacci(index - 1) + +And that code is in ``pFibA.py``. +In the repl we can measure its performance with ``timeit``: + +.. code-block:: bash + + >>> import pFibA + >>> pFibA.fibonacci(30) + 832040 + >>> import timeit + >>> ti_py = timeit.timeit(f'pFibA.fibonacci(30)', setup='import pFibA', number=10) + >>> print(f'Python timeit: {ti_py:8.6f}') + Python timeit: 1.459842 + >>> + +----------------------- +Faster Please +----------------------- + +Now we want something faster so we turn to creating a C extension. + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The C Equivalent Function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Firstly we can write the C equivalent to ``fibonacci()`` in the file ``cFibA.c``, note the inclusion of ``"Python.h"`` +which will give us access to the whole Python C API (we will use that later on): + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + #include "Python.h" + + long fibonacci(long index) { + if (index < 2) { + return index; + } + return fibonacci(index - 2) + fibonacci(index - 1); + } + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The Python Interface to C +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +So far we have a pure C function, we now write a C function that takes Python objects as arguments, converts them to C +objects (so-called 'un-boxing'), calls ``fibonacci()`` then converts the C result to a Python object +(so-called 'boxing'). + +.. code-block:: c + + static PyObject * + py_fibonacci(PyObject *Py_UNUSED(module), PyObject *args) { + long index; + + if (!PyArg_ParseTuple(args, "l", &index)) { + return NULL; + } + long result = fibonacci(index); + return Py_BuildValue("l", result); + } + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The Python Module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Then we need to write some C code that defines the Python module that contains this function. +The first is a data structure to define the Python functions in the module: + +.. code-block:: c + + static PyMethodDef module_methods[] = { + { + "fibonacci", + (PyCFunction) py_fibonacci, + METH_VARARGS, + "Returns the Fibonacci value." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +Then we have a structure that defines the module itself, its name and so on. +Note that this references the ``module_methods`` structure above: + +.. code-block:: c + + static PyModuleDef cFibA = { + PyModuleDef_HEAD_INIT, + .m_name = "cFibA", + .m_doc = "Fibonacci in C.", + .m_size = -1, + .m_methods = module_methods, + }; + +Lastly a function to to initialise the module: + +.. code-block:: c + + PyMODINIT_FUNC PyInit_cFibA(void) { + PyObject *m = PyModule_Create(&cFibA); + return m; + } + +.. index:: + single: setup.py + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The ``setup.py`` File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Finally we add a ``setup.py`` that specifies how to compile this code: + +.. code-block:: + + from setuptools import setup, Extension + + setup( + name='cPyExtPatt', + version='0.1.0', + author='AUTHOR', + description='Python Extension example.', + ext_modules=[ + Extension( + "cPyExtPatt.SimpleExample.cFibA", + sources=['src/cpy/SimpleExample/cFibA.c', ], + include_dirs=[], + library_dirs=[], + libraries=[], + extra_compile_args=[ + '-Wall', '-Wextra', '-Werror', '-Wfatal-errors', '-Wpedantic', + '-Wno-unused-function', '-Wno-unused-parameter', + '-Qunused-arguments', '-std=c99', + '-UDEBUG', '-DNDEBUG', '-Ofast', '-g', + ], + ) + ] + ) + +Running ``python setup.py develop`` will compile and build the module which can be used thus: + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Trying it Out +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + >>> from cPyExtPatt.SimpleExample import cFibA + >>> cFibA.fibonacci(8) + 21 + +Great, it works! + +---------------------------- +How Did We Do? +---------------------------- + +There is a test file that uses ``timeit`` to check the performance of the Python and C code at +``src/cpy/SimpleExample/timeit_test.py``. + + +.. code-block:: bash + + (PythonExtPatt3.11_A) SimpleExample git:(develop) $ python timeit_test.py + Index: 32 number of times: 20 + Version A, no cacheing: + Python timeit: 7.321638 + C timeit: 0.131115 + C is 55.8 times FASTER. + +So with a small bit of work we have got a performance improvement of 55x. + +---------------------------- +It's Not Over Yet +---------------------------- + +Suppose we change the Python code by adding a couple of lines thus that uses a local cache for the results. +We put this in the file ``pFibB.py``: + +.. code-block:: python + + import functools + + @functools.cache + def fibonacci(index: int) -> int: + if index < 2: + return index + return fibonacci(index - 2) + fibonacci(index - 1) + +Now what does our timing code say? + +.. code-block:: bash + + Version A with Python cache, no C cache: + Python timeit: 0.000012 + C timeit: 0.130394 + C is 11058.7 times SLOWER. + +So our Python code is now vastly faster than our C code. +This emphasises that performance can also come from a good choice of libraries, data structures, algorithms, +cacheing and other techniques as well as the choice of the language of the implementation. + +------------------------------- +C Fights Back +------------------------------- + +Whatever we can do in Python we can do in C so what if we write ``cFibB.c`` to change the ``fibonacci()`` function to +have a cache as well? + +.. code-block:: c + + long fibonacci(long index) { + static long *cache = NULL; + if (!cache) { + /* FIXME */ + cache = calloc(1000, sizeof(long)); + } + if (index < 2) { + return index; + } + if (!cache[index]) { + cache[index] = fibonacci(index - 2) + fibonacci(index - 1); + } + return cache[index]; + } + +Now what does our timeing code say? + +.. code-block:: bash + + Version B, both are cached: + Python timeit: 0.000004 + C timeit: 0.000007 + C is 1.9 times SLOWER. + +So our C code is back in the game but still slower. +What is more the C code has added significant complexity to our codebase. +And this codebase has to be maintained, and at what cost? +The C code has also added significant risk as well as identified by the ``/* FIXME */`` comment above. + +So while the options are available there are tradeoffs to be made. + +-------------------------- +Summary +-------------------------- + +- C Extensions can give vastly improved performance. +- A good choice of Python libraries, algorithms, code architecture and design can improve performance less + expensively than going to C. +- All of this exposes the possible tradeoffs between the techniques. +- It is very useful in software engineering to have tradeoffs such as these that are explicit and visible. + +Next up: understanding reference counts and Python's terminology. diff --git a/doc/sphinx/source/struct_sequence.rst b/doc/sphinx/source/struct_sequence.rst new file mode 100644 index 0000000..81ecb96 --- /dev/null +++ b/doc/sphinx/source/struct_sequence.rst @@ -0,0 +1,873 @@ +.. highlight:: python + :linenothreshold: 25 + +.. toctree:: + :maxdepth: 3 + +.. + Links, mostly to the Python documentation. + Specific container links are just before the appropriate section. + +.. _namedtuple: https://docs.python.org/3/library/collections.html#collections.namedtuple +.. _namedtuples: https://docs.python.org/3/library/collections.html#collections.namedtuple +.. _dataclass: https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass +.. _dataclasses: https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass + +.. _Struct Sequence API: https://docs.python.org/3/c-api/tuple.html#struct-sequence-objects +.. _Struct Sequence Object: https://docs.python.org/3/c-api/tuple.html#struct-sequence-objects +.. _Struct Sequence Objects: https://docs.python.org/3/c-api/tuple.html#struct-sequence-objects +.. _PyStructSequence_NewType(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_NewType +.. _PyStructSequence_InitType(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_InitType +.. _PyStructSequence_InitType2(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_InitType2 +.. _PyStructSequence_Desc: https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_Desc +.. _PyStructSequence_Field: https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_Field +.. _PyStructSequence_UnnamedField: https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_UnnamedField +.. _PyStructSequence_New(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_New +.. _PyStructSequence_GetItem(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_GetItem +.. _PyStructSequence_GET_ITEM(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_GET_ITEM +.. _PyStructSequence_SetItem(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_SetItem +.. _PyStructSequence_SET_ITEM(): https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_SET_ITEM + +.. _PyTypeObject: https://docs.python.org/3/c-api/type.html#c.PyTypeObject + +.. _chapter_struct_sequence: + +.. index:: + single: Struct Sequence + +================================================== +Struct Sequence (a ``namedtuple`` in C) +================================================== + +A `Struct Sequence Object`_ object is, more or less, the C equivalent of Python's `namedtuple`_ type [#]_. + +As a reminder here is how named tuples work in Python: + +.. code-block:: python + + >>> from collections import namedtuple + >>> nt_type = namedtuple('MyNamedTuple', ['field_one', 'field_two']) + >>> nt_type + + >>> nt_type._fields + ('field_one', 'field_two') + >>> nt = nt_type(['foo', 'bar']) + >>> nt.field_one + 'foo' + >>> nt.index('bar') + 1 + +The C `Struct Sequence API`_ allows you to define and create `Struct Sequence Objects`_ within C but act (almost) like +``collections.namedtuple`` objects. +These are very useful in creating the equivalent of a C ``struct`` in Python. + + +.. index:: + single: Struct Sequence; Differences from namedtuple + pair: Documentation Lacunae; Struct Sequence vs namedtuple + +------------------------------------------------------------------- +Differences Between a C Struct Sequence and a Python `namedtuple`_ +------------------------------------------------------------------- + +Unlike a Python `namedtuple`_ a C Struct Sequence does *not* have the following functions and attributes +(the official Python documentation does not point this out): + +- `_make() `_ +- `_asdict() `_ +- `_replace() `_ +- `_fields `_ +- `_field_defaults `_ + +`Struct Sequence Objects`_ also differ from `namedtuples`_ in the way that members can be accessed. +`namedtuples`_ can access *all* their members either by name or by index. +A `Struct Sequence Object`_ can be designed so that any attribute can be accessed by either name or index or both +(or even neither!). + +.. index:: + single: Struct Sequence; Basic Example + +------------------------------------------------------------------ +A Basic C Struct Sequence +------------------------------------------------------------------ + +Here is an example of defining a Struct Sequence in C (the code is in ``src/cpy/StructSequence/cStructSequence.c``). + +.. index:: + single: Struct Sequence; Documentation String + +Documentation String +-------------------- + +First create a named documentation string: + +.. code-block:: c + + PyDoc_STRVAR( + BasicNT_docstring, + "A basic named tuple type with two fields." + ); + +.. index:: + single: Struct Sequence; Field Specifications + +Field Specifications +-------------------- + +Now create the field definitions as an array of `PyStructSequence_Field`_. +These are just pairs of ``{field_name, field_description}``: + +.. code-block:: c + + static PyStructSequence_Field BasicNT_fields[] = { + {"field_one", "The first field of the named tuple."}, + {"field_two", "The second field of the named tuple."}, + {NULL, NULL} + }; + +.. index:: + single: Struct Sequence; Type Specification + +Struct Sequence Type Specification +---------------------------------- + +Now create the `PyStructSequence_Desc`_ that is a name, documentation, fields and the number of fields visible in +Python. +The latter value is explained later but for the moment make it the number of declared fields. + +.. code-block:: c + + static PyStructSequence_Desc BasicNT_desc = { + "cStructSequence.BasicNT", + BasicNT_docstring, + BasicNT_fields, + 2, + }; + +.. note:: + + If the given number of fields (`n_in_sequence`_) is greater than the length of the fields array then + `PyStructSequence_NewType()`_ will return NULL. + + There is a test example of this ``dbg_PyStructSequence_n_in_sequence_too_large()`` in + ``src/cpy/Containers/DebugContainers.c``. + +.. index:: + single: Struct Sequence; Creating an Instance + +Creating an Instance +-------------------- + +Here is a function ``BasicNT_create()`` that creates a Struct Sequence from arguments provided from a Python session. +Things to note: + +- There is a static `PyTypeObject`_ which holds a reference to the Struct Sequence type. +- This is initialised with `PyStructSequence_NewType()`_ that takes the ``BasicNT_desc`` described above. +- Then the function `PyStructSequence_New()`_ is used to create a new, empty, instance of that type. +- Finally `PyStructSequence_SetItem()`_ is used to set each individual field from the given arguments. + +.. note:: + + The careful use of ``Py_INCREF`` when setting the fields. + +.. code-block:: c + + static PyObject * + BasicNT_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + static PyTypeObject *static_BasicNT_Type = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &field_one, &field_two)) { + return NULL; + } + /* The two fields are PyObjects. If your design is that those arguments should be + * specific types then take the opportunity here to test that they are the + * expected types. + */ + + /* Now check that the type is available. */ + if (!static_BasicNT_Type) { + static_BasicNT_Type = PyStructSequence_NewType(&BasicNT_desc); + if (!static_BasicNT_Type) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a BasicNT type with PyStructSequence_NewType()" + ); + return NULL; + } + } + PyObject *result = PyStructSequence_New(static_BasicNT_Type); + if (!result) { + PyErr_SetString( + PyExc_MemoryError, + "Can not create a Struct Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created + * (i.e. its reference count is not increased)." + * + * So we increment as PyStructSequence_SetItem seals the reference otherwise + * if the callers arguments goes out of scope we will/may get undefined behaviour + * when accessing the named tuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + return result; + } + +This function is then added to the ``cStructSequence`` module like this: + +.. code-block:: c + + static PyMethodDef cStructSequence_methods[] = { + /* More stuff here... */ + { + "BasicNT_create", + (PyCFunction) BasicNT_create, + METH_VARARGS | METH_KEYWORDS, + "Create a BasicNT from the given values." + }, + /* More stuff here... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +Using an Instance +-------------------- + +And can be used like this: + +.. code-block:: python + + from cPyExtPatt import cStructSequence + + def test_basic_nt_create(): + basic_nt = cStructSequence.BasicNT_create('foo', 'bar') + assert str(type(basic_nt)) == "" + + + def test_basic_nt_create_attributes(): + basic_nt = cStructSequence.BasicNT_create('foo', 'bar') + assert basic_nt.field_one == "foo" + assert basic_nt.field_two == "bar" + assert basic_nt.index("foo") == 0 + assert basic_nt.index("bar") == 1 + assert basic_nt.n_fields == 2 + assert basic_nt.n_sequence_fields == 2 + assert basic_nt.n_unnamed_fields == 0 + + +------------------------------------------------- +Whether to Provide Access to the Type from Python +------------------------------------------------- + +One decision to be made is whether to expose your Struct Sequence *type* from the module. +There are only two use cases for this: + +- Do you want the user to be able to create your Struct Sequence/``namedtuple`` directly from Python? + In which case then you need to expose the type of your Struct Sequence from the module. + Then anyone can create these objects directly from Python. +- If the objects are created only in C then you do not need to expose the *type* in the module + but you can create functions the create those Python objects (and their types) dynamically. + +Firstly, exposing the Struct Sequence type to Python. + +.. index:: + single: Struct Sequence; Exposing the Type + +Exposing the Type from the CPython Module +----------------------------------------- + +In this case the Struct Sequence can be created *from* Python (as well as from C). + +For example here is a simple Struct Sequence ``cStructSequence.NTRegistered`` that contains two fields. + +First, creating the documentation: + +.. code-block:: c + + PyDoc_STRVAR( + NTRegistered_docstring, + "A namedtuple type with two fields that is" + "registered with the cStructSequence module." + ); + +Defining the fields, this is an array of `PyStructSequence_Field`_ which are pairs of +``{field_name, field_documentation}`` (both strings) and terminated with a NULL sentinel: + +.. code-block:: c + + static PyStructSequence_Field NTRegistered_fields[] = { + {"field_one", "The first field of the namedtuple."}, + {"field_two", "The second field of the namedtuple."}, + {NULL, NULL} + }; + +Creating the Struct Sequence description that will define the type. +This is a `PyStructSequence_Desc`_ that consists of: + +- The Struct Sequence name, this must include the module name so is of the form + ``"module_name.struct_sequence_name"``: +- The documentation string. +- The fields as an array of `PyStructSequence_Field`_. +- The number of fields exposed to Python. + +.. code-block:: c + + static PyStructSequence_Desc NTRegistered_desc = { + "cStructSequence.NTRegistered", + NTRegistered_docstring, + NTRegistered_fields, + 2, + }; + +Then the module initialisation code looks like this, this uses `PyStructSequence_NewType()`_ to create the type: + +.. code-block:: c + + PyMODINIT_FUNC + PyInit_cStructSequence(void) { + PyObject *m; + m = PyModule_Create(&cStructSequence_cmodule); + if (m == NULL) { + return NULL; + } + /* Initialise NTRegisteredType */ + PyObject *NTRegisteredType = (PyObject *) PyStructSequence_NewType( + &NTRegistered_desc + ); + if (NTRegisteredType == NULL) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(NTRegisteredType); + PyModule_AddObject(m, "NTRegisteredType", NTRegisteredType); + + /* + * Other module initialisation code here. + */ + + return m; + } + +This can be used thus in Python: + +.. code-block:: python + + from cPyExtPatt import cStructSequence + + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert str(nt) == "cStructSequence.NTRegistered(field_one='foo', field_two='bar')" + + +There are tests for this in ``tests/unit/test_c_struct_sequence.py``. + +.. index:: + single: Struct Sequence; Hiding the Type + +Hiding the Type in the Module +--------------------------------- + +In this case the Struct Sequence can be *not* be created from Python, only in C. +Even though the constructor is not accessible from Python the type is, as we will see. + +For example here is a simple Struct Sequence ``cStructSequence.NTUnRegistered`` that contains two fields. +It is, initially, very similar to the above. + +.. code-block:: c + + PyDoc_STRVAR( + NTUnRegistered_docstring, + "A namedtuple type with two fields that is" + " not registered with the cStructSequence module." + ); + + static PyStructSequence_Field NTUnRegistered_fields[] = { + {"field_one", "The first field of the namedtuple."}, + {"field_two", "The second field of the namedtuple."}, + {NULL, NULL} + }; + + static PyStructSequence_Desc NTUnRegistered_desc = { + "cStructSequence.NTUnRegistered", + NTUnRegistered_docstring, + NTUnRegistered_fields, + 2, + }; + +However as the type is not initialised in the module definition it remains static to the module C code. +It is, as yet, uninitialised: + +.. code-block:: c + + /* Type initailised dynamically by NTUnRegistered_create(). */ + static PyTypeObject *static_NTUnRegisteredType = NULL; + +This type, and objects created from it, can be made with a function call, +in this case taking variable and keyword arguments: + +.. code-block:: c + + /* A function that creates a cStructSequence.NTUnRegistered dynamically. */ + PyObject * + NTUnRegistered_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &field_one, &field_two)) { + return NULL; + } + /* The two fields are PyObjects. If your design is that those arguments should be + * specific types then take the opportunity here to test that they are the + * expected types. + */ + + /* Initialise the static static_NTUnRegisteredType. + * Note: PyStructSequence_NewType returns a new reference. + */ + if (!static_NTUnRegisteredType) { + static_NTUnRegisteredType = PyStructSequence_NewType(&NTUnRegistered_desc); + if (!static_NTUnRegisteredType) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a type with PyStructSequence_NewType()" + ); + return NULL; + } + } + PyObject *result = PyStructSequence_New(static_NTUnRegisteredType); + if (!result) { + PyErr_SetString( + PyExc_MemoryError, + "Can not create a Struct Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created + * (i.e. its reference count is not increased)." + * So we increment as PyStructSequence_SetItem seals the reference otherwise if + * the callers arguments go out of scope we will/may get undefined behaviour when + * accessing the namedtuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + return result; + } + +And this can be used thus: + +.. code-block:: python + + from cPyExtPatt import cStructSequence + + ntu = cStructSequence.NTUnRegistered_create('foo', 'bar') + # Note that the type is available + assert str(type(ntu)) == "" + +A common use of this is converting a C ``struct`` to a Python ``namedtuple``. + +.. index:: + single: Struct Sequence; C structs + +----------------------------------------- +Converting a C ``struct`` to a namedtuple +----------------------------------------- + +A common use case for *not* exposing the ``namedtuple`` type from the module is when the data object can *only* be +created in C. +Suppose that we have a simple struct representing a transaction. + +.. code-block:: c + + /** + * Representation of a simple transaction. + */ + struct cTransaction { + long id; /* The transaction id. */ + char *reference; /* The transaction reference. */ + double amount; /* The transaction amount. */ + }; + +And we have a C function that can recover a transaction given its ID: + +.. code-block:: c + + /** + * An example function that might recover a transaction from within C code, + * possibly a C library. + * In practice this will actually do something more useful that this function does! + * + * @param id The transaction ID. + * @return A struct cTransaction corresponding to the transaction ID. + */ + static struct cTransaction get_transaction(long id) { + struct cTransaction ret = {id, "Some reference.", 42.76}; + return ret; + } + +Then we create a ``namedtuple`` type that mirrors the C ``struct Transaction``: + +.. code-block:: c + + PyDoc_STRVAR( + cTransaction_docstring, + "Example of a named tuple type representing a transaction created in C." + " The type is not registered with the cStructSequence module." + ); + + static PyStructSequence_Field cTransaction_fields[] = { + {"id", "The transaction id."}, + {"reference", "The transaction reference."}, + {"amount", "The transaction amount."}, + {NULL, NULL} + }; + + static PyStructSequence_Desc cTransaction_desc = { + "cStructSequence.cTransaction", + cTransaction_docstring, + cTransaction_fields, + 3, + }; + +This Python type is declared static and initialised dynamically when necessary. +As this might be used by multiple functions so we give it an API: + +.. code-block:: c + + /* Type initialised dynamically by get_cTransactionType(). */ + static PyTypeObject *static_cTransactionType = NULL; + + static PyTypeObject *get_cTransactionType(void) { + if (static_cTransactionType == NULL) { + static_cTransactionType = PyStructSequence_NewType(&cTransaction_desc); + if (static_cTransactionType == NULL) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a cTransaction type with PyStructSequence_NewType()" + ); + return NULL; + } + } + return static_cTransactionType; + } + +Now the Python/C interface function: + +.. code-block:: c + + /* A function that creates a cStructSequence.NTUnRegistered dynamically. */ + PyObject * + cTransaction_get(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"id", NULL}; + long id = 0l; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "l", kwlist, &id)) { + return NULL; + } + PyObject *result = PyStructSequence_New(get_cTransactionType()); + if (!result) { + assert(PyErr_Occurred()); + return NULL; + } + + struct cTransaction transaction = get_transaction(id); + PyStructSequence_SetItem(result, 0, PyLong_FromLong(transaction.id)); + PyStructSequence_SetItem(result, 1, PyUnicode_FromString(transaction.reference)); + PyStructSequence_SetItem(result, 2, PyFloat_FromDouble(transaction.amount)); + return result; + } + +Add to the module methods: + +.. code-block:: c + + static PyMethodDef cStructSequence_methods[] = { + /* Other stuff... */ + { + "cTransaction_get", + (PyCFunction) cTransaction_get, + METH_VARARGS | METH_KEYWORDS, + "Example of getting a transaction." + }, + /* Other stuff... */ + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +And then this can be called from Python like this: + +.. code-block:: c + + nt = cStructSequence.cTransaction_get(17145) + assert nt.id == 17145 + assert nt.reference == "Some reference." + assert nt.amount == 42.76 + +.. _n_in_sequence: https://docs.python.org/3/c-api/tuple.html#c.PyStructSequence_Desc.n_in_sequence + +.. index:: + single: Struct Sequence; Controlling Member Access + +--------------------------------------------- +Controlling Member Access +--------------------------------------------- + +`Struct Sequence Objects`_ differ from `namedtuples`_ in the way that members can be accessed. +A `Struct Sequence Object`_ can be designed so that any attribute can be accessed by either name or index or both +(or even neither!). +This describes how to do this. + +.. index:: + single: Struct Sequence; n_in_sequence + +The Importance of the ``n_in_sequence`` Field +--------------------------------------------- + +`PyStructSequence_Desc`_ has a field `n_in_sequence`_ which needs some explaining (the Python documentation is pretty +silent on this). +Normally `n_in_sequence`_ is equal to the number of fields, however what happens if it is not? + +``n_in_sequence`` > Number of Fields +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As mentioned above if the given number of fields (`n_in_sequence`_) is greater than the length of the fields array then +`PyStructSequence_NewType()`_ will return NULL. + +There is a test example of this ``dbg_PyStructSequence_n_in_sequence_too_large()`` in +``src/cpy/Containers/DebugContainers.c``. + +``n_in_sequence`` < Number of Fields +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this case the members with an index >= `n_in_sequence`_ will raise an ``IndexError``. +However that same member can always be accessed from Python by name. + +There some illustrative tests ``test_excess_nt_*`` in ``tests/unit/test_c_struct_sequence.py`` for this. + +.. index:: + single: Struct Sequence; Unnamed Fields + single: Struct Sequence; PyStructSequence_UnnamedField + pair: Documentation Lacunae; Struct Sequence Unnamed Fields + +--------------------------------------------- +Unnamed Fields +--------------------------------------------- + +The Struct Sequence C API provides for having *unnamed* fields. +These are fields unavailable to Python code but available to C code. +The rules using these are quite involved and the Python documentation is poor at describing them so I'll do my best +here. + +.. note:: + + Unnamed fields does not appear to work in Python versions prior to 3.11 with the following compile time error: + + ``E ImportError: dlopen(... cStructSequence.cpython-310-darwin.so, 0x0002): symbol not found in flat namespace '_PyStructSequence_UnnamedField'`` + +The rules appear to be: + +* Unnamed fields in the `PyStructSequence_Field`_ *must* follow named fields. +* Named fields in the `PyStructSequence_Field`_ can *not* follow unnamed fields. +* The ``n_in_sequence`` in the type `PyStructSequence_Desc`_ must be <= the number of *named* fields. + +Failing to follow the rules will cause a ``SystemError`` when you use ``repr()`` or ``str()`` such as: + +.. code-block:: + + SystemError: In structseq_repr(), member 2 name is NULL for type module.struct_sequence_simple_with_unnamed_field + + +`PyStructSequence_UnnamedField`_ is the sentinel that describes the 'name' of the unnamed field. +However you can not use it directly when declaring a `PyStructSequence_Field`_: + +.. code-block:: c + + static PyStructSequence_Field NTWithUnnamedField_fields[] = { + {"field_one", "The first field of the named tuple."}, + {PyStructSequence_UnnamedField, "Documentation for an unnamed field."}, + {NULL, NULL} + }; + +This will fail with a compilation error along the lines of: + +.. code-block:: text + + cStructSequence.c:391:10: fatal error: initializer element is not a compile-time constant + {PyStructSequence_UnnamedField, "Documentation for an unnamed field."}, + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 1 error generated. + +Instead declare the field name as NULL: + +.. code-block:: c + + static PyStructSequence_Field NTWithUnnamedField_fields[] = { + {"field_one", "The first field of the named tuple."}, + /* Use NULL then replace with PyStructSequence_UnnamedField + * otherwise get an error "initializer element is not a compile-time constant" */ + {NULL, "Documentation for an unnamed field."}, + {NULL, NULL} + }; + +Then before the type is used the ``NULL`` needs to be replaced with `PyStructSequence_UnnamedField`_. +Here is the complete code that demonstrates initialising a static Struct Sequence type +``static_NTWithUnnamedField_Type``. + +Field Declaration +----------------- + +First declare the fields: + +.. code-block:: c + + static PyStructSequence_Field NTWithUnnamedField_fields[] = { + {"field_one", "The first field of the named tuple."}, + /* Use NULL then replace with PyStructSequence_UnnamedField + * otherwise get an error "initializer element is not a compile-time constant" */ + {NULL, "Documentation for an unnamed field."}, + {NULL, NULL} + }; + +Then the field description: + +.. code-block:: c + + PyDoc_STRVAR( + NTWithUnnamedField_docstring, + "A basic named tuple type with an unnamed field." + ); + + static PyStructSequence_Desc NTWithUnnamedField_desc = { + "cStructSequence.NTWithUnnamedField", + NTWithUnnamedField_docstring, + NTWithUnnamedField_fields, + 1, /* Of two fields only one available to Python by name. */ + }; + +Type Declaration +---------------- + +Now the Struct Sequence type, for convenience the static is accessed with a *getter* that also serves as an +initialisation function: + +.. code-block:: c + + static PyTypeObject *static_NTWithUnnamedField_Type = NULL; + + /** + * Initialises and returns the \c NTWithUnnamedField_Type. + * @return The initialised type. + */ + static PyTypeObject *get_NTWithUnnamedField_Type(void) { + if (!static_NTWithUnnamedField_Type) { + /* Substitute PyStructSequence_UnnamedField for NULL. */ + NTWithUnnamedField_fields[1].name = PyStructSequence_UnnamedField; + /* Create and initialise the type. */ + static_NTWithUnnamedField_Type = PyStructSequence_NewType( + &NTWithUnnamedField_desc + ); + if (!static_NTWithUnnamedField_Type) { + PyErr_SetString( + PyExc_RuntimeError, + "Can not initialise a NTWithUnnamedField type with PyStructSequence_NewType()" + ); + return NULL; + } + } + return static_NTWithUnnamedField_Type; + } + +Struct Sequence Creation +------------------------ + +Here is the constructor that takes three arguments: + +.. code-block:: c + + static PyObject * + NTWithUnnamedField_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", "field_three", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + PyObject *field_three = NULL; + + if ( + !PyArg_ParseTupleAndKeywords( + args, kwds, "OOO", kwlist, &field_one, &field_two, &field_three + ) + ) { + return NULL; + } + /* The three fields are PyObjects. + * If your design is that those arguments should be specific types + * then take the opportunity here to test that they are the expected types. + */ + + PyObject *result = PyStructSequence_New(get_NTWithUnnamedField_Type()); + if (!result) { + PyErr_SetString( + PyExc_RuntimeError, + "Can not create a NTWithUnnamedField Struct" + " Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created + * (i.e. its reference count is not increased)." + * So we increment as PyStructSequence_SetItem seals the reference otherwise + * if the callers arguments go out of scope we will/may get undefined + * behaviour when accessing the named tuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + Py_INCREF(field_three); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + PyStructSequence_SetItem(result, 2, field_three); + assert(!PyErr_Occurred()); + return result; + } + +Access from Python +------------------ + +Once built this can be accessed from Python (see ``tests/unit/test_c_struct_sequence.py``): + +.. code-block:: python + + from cPyExtPatt import cStructSequence + + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + # All these tests are True + assert len(ntuf) == 1 + assert ntuf.n_fields == 2 + assert ntuf.n_sequence_fields == 1 + assert ntuf.n_unnamed_fields == 1 + assert tuple(ntuf) == ('foo',) + assert ntuf[0] == 'foo' + # Will raise an IndexError: 'tuple index out of range' + with pytest.raises(IndexError) as err: + assert ntuf[1] == 'bar' + assert repr(ntuf) == "cStructSequence.NTWithUnnamedField(field_one='foo')" + assert str(ntuf) == "cStructSequence.NTWithUnnamedField(field_one='foo')" + +.. Example footnote [#]_. + +.. rubric:: Footnotes + +.. [#] `namedtuples`_ have been largely superseded by `dataclasses`_ and the is no direct C equivalent for them. diff --git a/doc/sphinx/source/subclassing_and_super_call.rst b/doc/sphinx/source/subclassing_and_super_call.rst new file mode 100644 index 0000000..bc5b585 --- /dev/null +++ b/doc/sphinx/source/subclassing_and_super_call.rst @@ -0,0 +1,752 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 3 + +.. _chapter_subclassing_and_using_super: + +.. index:: + single: Subclassing + single: Subclassing; Using super() + +************************************** +Subclassing and Using ``super()`` +************************************** + +This chapter describes how to subclass existing types and how to call ``super()`` in C where necessary. + +.. index:: + single: Subclassing; Basic + +================================= +Basic Subclassing +================================= + +In this example we are going to subclass the built in ``list`` object and just add a new attribute ``state``. +The code is based on +`Python documentation on subclassing `_ +The full code is in ``src/cpy/SubClass/sublist.c`` and the tests are in ``tests/unit/test_c_subclass.py``. + +----------------------------- +Writing the C Extension +----------------------------- + +First the declaration of the ``SubListObject``: + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + #include + #include "structmember.h" + + typedef struct { + PyListObject list; // The superclass. + int state; // Our new attribute. + } SubListObject; + +The ``__init__`` method: + +.. code-block:: c + + static int + SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) { + return -1; + } + self->state = 0; + return 0; + } + +Now add an ``increment()`` method and the method table: + +.. code-block:: c + + static PyObject * + SubList_increment(SubListObject *self, PyObject *Py_UNUSED(unused)) { + self->state++; + return PyLong_FromLong(self->state); + } + + static PyMethodDef SubList_methods[] = { + {"increment", (PyCFunction) SubList_increment, METH_NOARGS, + PyDoc_STR("increment state counter")}, + {NULL, NULL, 0, NULL}, + }; + +Add the ``state`` attribute: + +.. code-block:: c + + static PyMemberDef SubList_members[] = { + {"state", T_INT, offsetof(SubListObject, state), 0, + "Value of the state."}, + {NULL, 0, 0, 0, NULL} /* Sentinel */ + }; + +Declare the type. + +Note that we do not initialise ``tp_base`` just yet. +The reason is that C99 requires the initializers to be “address constants”. +This is best described in the +`tp_base documentation `_. + +.. code-block:: c + + static PyTypeObject SubListType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "sublist.SubList", + .tp_doc = PyDoc_STR("SubList objects"), + .tp_basicsize = sizeof(SubListObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_init = (initproc) SubList_init, + .tp_methods = SubList_methods, + .tp_members = SubList_members, + }; + +Declare the module: + +.. code-block:: c + + static PyModuleDef sublistmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "sublist", + .m_doc = "Module that contains a subclass of a list.", + .m_size = -1, + }; + +Initialise the module, this is where we set ``tp_base``: + +.. code-block:: c + + PyMODINIT_FUNC + PyInit_sublist(void) { + PyObject *m; + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return NULL; + } + m = PyModule_Create(&sublistmodule); + if (m == NULL) { + return NULL; + } + Py_INCREF(&SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; + } + +----------------------------- +Setup and Build +----------------------------- + +In the ``setup.py`` add an Extension such as: + +.. code-block:: python + + Extension(name=f"cPyExtPatt.SubClass.sublist", + include_dirs=[ + '/usr/local/include', + ], + sources=[ + "src/cpy/SubClass/sublist.c", + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +The extension can be built with ``python setup.py develop``. + +----------------------------- +Test +----------------------------- + +The extension can used like this: + +.. code-block:: python + + from cPyExtPatt.SubClass import sublist + + obj = sublist.SubList() + assert obj.state == 0 + obj.increment() + assert obj.state == 1 + +This is fine for subclasses that just add some additional functionality however if you want to overload the super class +you need to be able to call ``super()`` from C which is described next. + +.. index:: + single: Subclassing; Calling super() from C + +================================= +Calling ``super()`` from C +================================= + +I needed to call super() from a C extension and I couldn't find a good description of how to do this online so I am +including this here. +The ability to call ``super()`` is needed when you want to modify the behaviour of inherited classes. + +Suppose we wanted to subclass a list and record how many times ``append()`` was called. +This is simple enough in pure Python: + +.. code-block:: python + + class SubList(list): + def __init__(self, *args, **kwargs): + self.appends = 0 + super().__init__(*args, **kwargs) + + def append(self, v): + self.appends += 1 + return super().append(v) + +To do it in C is a bit trickier. Taking as our starting point the +`example of sub-classing a list `_ +in the Python documentation (amended a little bit for our example). + +Our type contains an integer count of the number of appends. +That is set to zero on construction and can be accessed like a normal member. + +.. code-block:: c + + typedef struct { + PyListObject list; + int appends; + } SubListObject; + + + static int + SubListObject_init(SubListObject *self, PyObject *args, PyObject *kwds) + { + if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0) { + return -1; + } + self->appends = 0; + return 0; + } + + static PyMemberDef SubListObject_members[] = { + ... + {"appends", T_INT, offsetof(SubListObject, appends), 0, + "Number of append operations."}, + ... + {NULL, 0, 0, 0, NULL} /* Sentinel */ + }; + +We now need to create the ``append()`` function, which will call the superclass ``append()`` and then increment the +``appends`` counter: + +.. code-block:: c + + static PyMethodDef SubListObject_methods[] = { + ... + {"append", (PyCFunction)SubListObject_append, METH_VARARGS, + PyDoc_STR("Append to the list")}, + ... + {NULL, NULL, 0, NULL}, + }; + +This is where it gets tricky, how do we implement ``SubListObject_append``? + +-------------------------- +The Obvious Way is Wrong +-------------------------- + +A first attempt might do something like a method call on the ``PyListObject``: + +.. code-block:: c + + typedef struct { + PyListObject list; + int appends; + } SubListObject; + + /* Other stuff here. */ + + static PyObject * + SubListObject_append(SubListObject *self, PyObject *args) { + PyObject *result = PyObject_CallMethod((PyObject *)&self->list, "append", "O", args); + if (result) { + self->appends++; + } + return result; + } + +This leads to infinite recursion as the address of the first element of a C struct (``list``) is the address of the +struct so ``self`` is the same as ``&self->list``. This function is recursive with no base case. + +-------------------------- +Doing it Right +-------------------------- + +Our append method needs to use `super` to search our super-classes for the "append" method and call that. + +Here are a couple of ways of calling ``super()`` correctly: + +* Construct a ``super`` object directly and call that. +* Extract the ``super`` object from the builtins module and call that. + +The full code is in ``src/cpy/Util/py_call_super.h`` and ``src/cpy/Util/py_call_super.c``. + +.. index:: + single: Subclassing; Directly Calling super() from C + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Construct a ``super`` object directly +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The plan is to do this: + +* Create the arguments to initialise an instance of the class ``super``. +* Call ``super.__new__`` with those arguments. +* Call ``super.__init__`` with those arguments. +* With that ``super`` object then search for the method we want to call. + This is ``append`` in our case. + This calls the ``super_getattro`` method that performs the search and returns the Python function. +* Call that Python function and return the result. + +Our function is defined thus, for simplicity there is no error checking here. For the full function see below: + +.. code-block:: c + + /* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation creates a new super object on each call. + */ + PyObject * + call_super_pyname(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *super_func = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + PyObject *result = NULL; + + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + super_func = PyType_GenericNew(&PySuper_Type, super_args, NULL); + // Use tuple as first arg, super() second arg (i.e. kwargs) should be NULL + super_func->ob_type->tp_init(super_func, super_args, NULL); + func = PyObject_GetAttr(super_func, func_name); + result = PyObject_Call(func, args, kwargs); + Py_XDECREF(super_func); + Py_XDECREF(super_args); + Py_XDECREF(func); + return result; + } + +We can make this function quite general to be used in the CPython type system. +For convenience we can create two functions, one calls the super function by a C NTS, the other by a PyObject string. +The following code is essentially the same as above but with error checking. + +The header file might be py_call_super.h which just declares our two functions: + +.. code-block:: c + + #ifndef __PythonSubclassList__py_call_super__ + #define __PythonSubclassList__py_call_super__ + + #include + + extern PyObject * + call_super_pyname(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs); + extern PyObject * + call_super_name(PyObject *self, const char *func_cname, + PyObject *args, PyObject *kwargs); + + #endif /* defined(__PythonSubclassList__py_call_super__) */ + +And the implementation file would be py_call_super.c, this is the code above with full error checking: + +.. code-block:: c + + /* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation creates a new super object on each call. + */ + PyObject * + call_super_pyname(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *super_func = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + PyObject *result = NULL; + + // Error check input + if (!PyUnicode_Check(func_name)) { + PyErr_Format(PyExc_TypeError, + "super() must be called with unicode attribute not %s", + Py_TYPE(func_name)->tp_name); + } + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + if (!super_args) { + Py_DECREF(self->ob_type); + Py_DECREF(self); + PyErr_SetString(PyExc_RuntimeError, "Could not create arguments for super()."); + goto except; + } + super_func = PyType_GenericNew(&PySuper_Type, super_args, NULL); + if (!super_func) { + PyErr_SetString(PyExc_RuntimeError, "Could not create super()."); + goto except; + } + // Use tuple as first arg, super() second arg (i.e. kwargs) should be NULL + super_func->ob_type->tp_init(super_func, super_args, NULL); + if (PyErr_Occurred()) { + goto except; + } + func = PyObject_GetAttr(super_func, func_name); + if (!func) { + assert(PyErr_Occurred()); + goto except; + } + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_AttributeError, + "super() attribute \"%S\" is not callable.", func_name); + goto except; + } + result = PyObject_Call(func, args, kwargs); + if (!result) { + assert(PyErr_Occurred()); + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(result); + result = NULL; + finally: + Py_XDECREF(super_func); + Py_XDECREF(super_args); + Py_XDECREF(func); + return result; + } + + +.. index:: + single: Subclassing; Calling super() From builtins + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Extract the ``super`` object from the builtins +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Another way to do this is to fish out the `super` class from the builtins module and use that. +Incidentally this is how Cython does it. + +The steps are: + +#. Get the `builtins` module. +#. Get the `super` class from the `builtins` module. +#. Create a tuple of the arguments to pass to the super class. +#. Create the `super` object with the arguments. +#. Use this `super` object to call the function with the appropriate function arguments. + +Again this code has no error checking for simplicity: + +.. code-block:: c + + extern PyObject * + call_super_pyname_lookup(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *result = NULL; + PyObject *builtins = NULL; + PyObject *super_type = NULL; + PyObject *super = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + + builtins = PyImport_AddModule("builtins"); + // Borrowed reference + Py_INCREF(builtins); + super_type = PyObject_GetAttrString(builtins, "super"); + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + super = PyObject_Call(super_type, super_args, NULL); + // The following code is the same as call_super_pyname() + func = PyObject_GetAttr(super, func_name); + result = PyObject_Call(func, args, kwargs); + Py_XDECREF(builtins); + Py_XDECREF(super_args); + Py_XDECREF(super_type); + Py_XDECREF(super); + Py_XDECREF(func); + return result; + } + +Here is the function with full error checking: + +.. code-block:: c + + /* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation uses the builtin super(). + */ + extern PyObject * + call_super_pyname_lookup(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *result = NULL; + PyObject *builtins = NULL; + PyObject *super_type = NULL; + PyObject *super = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + + builtins = PyImport_AddModule("builtins"); + if (!builtins) { + assert(PyErr_Occurred()); + goto except; + } + // Borrowed reference + Py_INCREF(builtins); + super_type = PyObject_GetAttrString(builtins, "super"); + if (!super_type) { + assert(PyErr_Occurred()); + goto except; + } + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + if (!super_args) { + Py_DECREF(self->ob_type); + Py_DECREF(self); + PyErr_SetString(PyExc_RuntimeError, "Could not create arguments for super()."); + goto except; + } + super = PyObject_Call(super_type, super_args, NULL); + if (!super) { + assert(PyErr_Occurred()); + goto except; + } + // The following code is the same as call_super_pyname() + func = PyObject_GetAttr(super, func_name); + if (!func) { + assert(PyErr_Occurred()); + goto except; + } + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_AttributeError, + "super() attribute \"%S\" is not callable.", func_name); + goto except; + } + result = PyObject_Call(func, args, kwargs); + if (!result) { + assert(PyErr_Occurred()); + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(result); + result = NULL; + finally: + Py_XDECREF(builtins); + Py_XDECREF(super_args); + Py_XDECREF(super_type); + Py_XDECREF(super); + Py_XDECREF(func); + return result; + } + + +.. index:: + single: Subclassing; With Overloading + +===================================== +Subclassing with Overloading +===================================== + +Lets revisit our subclass of the builtin ``list`` that counts how many time ``append()`` is called and we will use +the C ``super()`` API described above. + +The full code is in ``src/cpy/SubClass/sublist.c`` and the tests are in ``tests/unit/test_c_subclass.py``. + +----------------------------- +Writing the C Extension +----------------------------- + +First the declaration and initialisation of ``SubListObject`` that has an ``appends`` counter. +Note the inclusion of ``py_call_super.h``: + +.. code-block:: c + + #define PY_SSIZE_T_CLEAN + #include + #include "structmember.h" + + #include "py_call_super.h" + + typedef struct { + PyListObject list; + int appends; + } SubListObject; + + static int + SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) { + return -1; + } + self->appends = 0; + return 0; + } + +Now the implementation of ``append`` that makes the ``super()`` call and then increments the ``appends`` attribute +and returning the value of the ``super()`` call: + +.. code-block:: c + + static PyObject * + SubList_append(SubListObject *self, PyObject *args) { + PyObject *result = call_super_name((PyObject *)self, "append", + args, NULL); + if (result) { + self->appends++; + } + return result; + } + +The declaration of methods, members and the type: + +.. code-block:: c + + static PyMethodDef SubList_methods[] = { + {"append", (PyCFunction) SubList_append, METH_VARARGS, + PyDoc_STR("append an item")}, + {NULL, NULL, 0, NULL}, + }; + + static PyMemberDef SubList_members[] = { + {"appends", T_INT, offsetof(SubListObject, appends), 0, + "Number of append operations."}, + {NULL, 0, 0, 0, NULL} /* Sentinel */ + }; + + static PyTypeObject SubListType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "sublist.SubList", + .tp_doc = PyDoc_STR("SubList objects"), + .tp_basicsize = sizeof(SubListObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_init = (initproc) SubList_init, + .tp_methods = SubList_methods, + .tp_members = SubList_members, + }; + +Finally the module definition which is very similar to before: + +.. code-block:: c + + static PyModuleDef sublistmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "sublist", + .m_doc = "Module that contains a subclass of a list.", + .m_size = -1, + }; + + PyMODINIT_FUNC + PyInit_sublist(void) { + PyObject *m; + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return NULL; + } + m = PyModule_Create(&sublistmodule); + if (m == NULL) { + return NULL; + } + Py_INCREF(&SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; + } + +----------------------------- +Setup and Build +----------------------------- + +In the ``setup.py`` add an Extension such as: + +.. code-block:: python + + Extension(name=f"cPyExtPatt.SubClass.sublist", + include_dirs=[ + '/usr/local/include', + ], + sources=[ + "src/cpy/SubClass/sublist.c", + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +The extension can be built with ``python setup.py develop``. + +----------------------------- +Test +----------------------------- + +The extension can used like this: + +.. code-block:: python + + from cPyExtPatt.SubClass import sublist + + obj = sublist.SubList() + assert obj.appends == 0 + obj.append(42) + assert obj.appends == 1 + assert obj == [42, ] + + +.. index:: + single: Subclassing; datetime Example + +-------------------------------------- +Another Example of Using this API +-------------------------------------- + +Here is a real example of using this see overloading ``replace()`` when subclassing a ``datetime`` in +:ref:`chapter_capsules_using_an_existing_capsule` from the chapter :ref:`chapter_capsules`. + +The code here calls the ``super()`` function then raises if the given arguments are unacceptable (trying to set the +``tzinfo`` property to ``None``): + +.. code-block:: c + + static PyObject * + DateTimeTZ_replace(PyObject *self, PyObject *args, PyObject *kwargs) { + PyObject *result = call_super_name(self, "replace", args, kwargs); + if (result) { + result = (PyObject *) raise_if_no_tzinfo((DateTimeTZ *) result); + } + return result; + } diff --git a/doc/sphinx/source/super_call.rst b/doc/sphinx/source/super_call.rst deleted file mode 100644 index c0041a8..0000000 --- a/doc/sphinx/source/super_call.rst +++ /dev/null @@ -1,345 +0,0 @@ -.. highlight:: python - :linenothreshold: 10 - -.. toctree:: - :maxdepth: 3 - -================================= -Calling ``super()`` from C -================================= - -I needed to call super() from a C extension and I couldn't find a good description of how to do this online so I am including this here. - -TODO: This code is specific to Python 3, add Python 2 support. - -Suppose we wanted to subclass a list and record how many times ``append()`` was called. This is simple enough in pure Python: - -.. code-block:: python - - class SubList(list): - def __init__(self, *args, **kwargs): - self.appends = 0 - super().__init__(*args, **kwargs) - - def append(self, v): - self.appends += 1 - return super().append(v) - -To do it in C is a bit trickier. Taking as our starting point the `example of sub-classing a list `_ in the Python documentation, amended a little bit for our example. - -Our type contains an integer count of the number of appends. That is set to zero on construction and can be accesssed like a normal member. - -.. code-block:: c - - typedef struct { - PyListObject list; - int appends; - } Shoddy; - - - static int - Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) - { - if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0) { - return -1; - } - self->appends = 0; - return 0; - } - - static PyMemberDef Shoddy_members[] = { - ... - {"appends", T_INT, offsetof(Shoddy, appends), 0, - "Number of append operations."}, - ... - {NULL, 0, 0, 0, NULL} /* Sentinel */ - }; - -We now need to create the ``append()`` function, this function will call the superclass ``append()`` and increment the ``appends`` counter: - -.. code-block:: c - - static PyMethodDef Shoddy_methods[] = { - ... - {"append", (PyCFunction)Shoddy_append, METH_VARARGS, - PyDoc_STR("Append to the list")}, - ... - {NULL, NULL, 0, NULL}, - }; - -This is where it gets tricky, how do we implement ``Shoddy_append``? - --------------------------- -The Obvious Way is Wrong --------------------------- - -A first attempt might do something like a method call on the ``PyListObject``: - -.. code-block:: c - - typedef struct { - PyListObject list; - int appends; - } Shoddy; - - /* Other stuff here. */ - - static PyObject * - Shoddy_append(Shoddy *self, PyObject *args) { - PyObject *result = PyObject_CallMethod((PyObject *)&self->list, "append", "O", args); - if (result) { - self->appends++; - } - return result; - } - -This leads to infinite recursion as the address of the first element of a C struct (``list``) is the address of the struct so ``self`` is the same as ``&self->list``. This function is recursive with no base case. - --------------------------- -Doing it Right --------------------------- - -Our append method needs to use `super` to search our super-classes for the "append" method and call that. - -Here are a couple of ways of calling ``super()`` correctly: - -* Construct a ``super`` object directly and call that. -* Extract the ``super`` object from the builtins module and call that. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Construct a ``super`` object directly -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The plan is to do this: - -* Create the arguments to initialise an instance of the class ``super``. -* Call ``super.__new__`` with those arguments. -* Call ``super.__init__`` with those arguments. -* With that ``super`` object then search for the method we want to call. This is ``append`` in our case. This calls the ``super_getattro`` method that performs the search and returns the Python function. -* Call that Python function and return the result. - -Our function is defined thus, for simplicity there is no error checking here. For the full function see below: - -.. code-block:: c - - PyObject * - call_super_pyname(PyObject *self, PyObject *func_name, PyObject *args, PyObject *kwargs) { - PyObject *super = NULL; - PyObject *super_args = NULL; - PyObject *func = NULL; - PyObject *result = NULL; - - // Create the arguments for super() - super_args = PyTuple_New(2); - Py_INCREF(self->ob_type); // Py_INCREF(&ShoddyType); in our specific case - PyTuple_SetItem(super_args, 0, (PyObject*)self->ob_type)); // PyTuple_SetItem(super_args, 0, (PyObject*)&ShoddyType) in our specific case - Py_INCREF(self); - PyTuple_SetItem(super_args, 1, self)); - // Creat the class super() - super = PyType_GenericNew(&PySuper_Type, super_args, NULL); - // Instantiate it with the tuple as first arg, no kwargs passed to super() so NULL - super->ob_type->tp_init(super, super_args, NULL); - // Use super to find the 'append' method - func = PyObject_GetAttr(super, func_name); - // Call that method - result = PyObject_Call(func, args, kwargs); - Py_XDECREF(super); - Py_XDECREF(super_args); - Py_XDECREF(func); - return result; - } - -We can make this function quite general to be used in the CPython type system. For convenience we can create two functions, one calls the super function by a C NTS, the other by a PyObject string. The following code is essentially the same as above but with error checking. - -The header file might be py_call_super.h which just declares our two functions: - -.. code-block:: c - - #ifndef __PythonSubclassList__py_call_super__ - #define __PythonSubclassList__py_call_super__ - - #include - - extern PyObject * - call_super_pyname(PyObject *self, PyObject *func_name, - PyObject *args, PyObject *kwargs); - extern PyObject * - call_super_name(PyObject *self, const char *func_cname, - PyObject *args, PyObject *kwargs); - - #endif /* defined(__PythonSubclassList__py_call_super__) */ - -And the implementation file would be py_call_super.c, this is the code above with full error checking: - -.. code-block:: c - - PyObject * - call_super_pyname(PyObject *self, PyObject *func_name, - PyObject *args, PyObject *kwargs) { - PyObject *super = NULL; - PyObject *super_args = NULL; - PyObject *func = NULL; - PyObject *result = NULL; - - if (! PyUnicode_Check(func_name)) { - PyErr_Format(PyExc_TypeError, - "super() must be called with unicode attribute not %s", - func_name->ob_type->tp_name); - } - - super_args = PyTuple_New(2); - // Py_INCREF(&ShoddyType); - Py_INCREF(self->ob_type); - // if (PyTuple_SetItem(super_args, 0, (PyObject*)&ShoddyType)) { - if (PyTuple_SetItem(super_args, 0, (PyObject*)self->ob_type)) { - assert(PyErr_Occurred()); - goto except; - } - Py_INCREF(self); - if (PyTuple_SetItem(super_args, 1, self)) { - assert(PyErr_Occurred()); - goto except; - } - - super = PyType_GenericNew(&PySuper_Type, super_args, NULL); - if (! super) { - PyErr_SetString(PyExc_RuntimeError, "Could not create super()."); - goto except; - } - // Make tuple as first arg, second arg (i.e. kwargs) should be NULL - super->ob_type->tp_init(super, super_args, NULL); - if (PyErr_Occurred()) { - goto except; - } - func = PyObject_GetAttr(super, func_name); - if (! func) { - assert(PyErr_Occurred()); - goto except; - } - if (! PyCallable_Check(func)) { - PyErr_Format(PyExc_AttributeError, - "super() attribute \"%S\" is not callable.", func_name); - goto except; - } - result = PyObject_Call(func, args, kwargs); - assert(! PyErr_Occurred()); - goto finally; - except: - assert(PyErr_Occurred()); - Py_XDECREF(result); - result = NULL; - finally: - Py_XDECREF(super); - Py_XDECREF(super_args); - Py_XDECREF(func); - return result; - } - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Extract the ``super`` object from the builtins -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Another way to do this is to fish out the `super` class from the builtins module and use that. Incidentially this is how Cython does it. - -The steps are: - -#. Get the `builtins` module. -#. Get the `super` class from the `builtins` module. -#. Create a tuple of the arguments to pass to the super class. -#. Create the `super` object with the arguments. -#. Use this `super` object to call the function with the appropriate function arguments. - -Again this code has no error checking for simplicity: - -.. code-block:: c - - extern PyObject * - call_super_pyname_lookup(PyObject *self, PyObject *func_name, - PyObject *args, PyObject *kwargs) { - PyObject *builtins = PyImport_AddModule("builtins"); - // Borrowed reference - Py_INCREF(builtins); - PyObject *super_type = PyObject_GetAttrString(builtins, "super"); - PyObject *super_args = PyTuple_New(2); - Py_INCREF(self->ob_type); - PyTuple_SetItem(super_args, 0, (PyObject*)self->ob_type); - Py_INCREF(self); - PyTuple_SetItem(super_args, 1, self); - PyObject *super = PyObject_Call(super_type, super_args, NULL); - PyObject *func = PyObject_GetAttr(super, func_name); - PyObject *result = PyObject_Call(func, args, kwargs); - Py_XDECREF(builtins); - Py_XDECREF(super_args); - Py_XDECREF(super_type); - Py_XDECREF(super); - Py_XDECREF(func); - return result; - } - -Here is the function with full error checking: - -.. code-block:: c - - extern PyObject * - call_super_pyname_lookup(PyObject *self, PyObject *func_name, - PyObject *args, PyObject *kwargs) { - PyObject *result = NULL; - PyObject *builtins = NULL; - PyObject *super_type = NULL; - PyObject *super = NULL; - PyObject *super_args = NULL; - PyObject *func = NULL; - - builtins = PyImport_AddModule("builtins"); - if (! builtins) { - assert(PyErr_Occurred()); - goto except; - } - // Borrowed reference - Py_INCREF(builtins); - super_type = PyObject_GetAttrString(builtins, "super"); - if (! super_type) { - assert(PyErr_Occurred()); - goto except; - } - super_args = PyTuple_New(2); - Py_INCREF(self->ob_type); - if (PyTuple_SetItem(super_args, 0, (PyObject*)self->ob_type)) { - assert(PyErr_Occurred()); - goto except; - } - Py_INCREF(self); - if (PyTuple_SetItem(super_args, 1, self)) { - assert(PyErr_Occurred()); - goto except; - } - super = PyObject_Call(super_type, super_args, NULL); - if (! super) { - assert(PyErr_Occurred()); - goto except; - } - func = PyObject_GetAttr(super, func_name); - if (! func) { - assert(PyErr_Occurred()); - goto except; - } - if (! PyCallable_Check(func)) { - PyErr_Format(PyExc_AttributeError, - "super() attribute \"%S\" is not callable.", func_name); - goto except; - } - result = PyObject_Call(func, args, kwargs); - assert(! PyErr_Occurred()); - goto finally; - except: - assert(PyErr_Occurred()); - Py_XDECREF(result); - result = NULL; - finally: - Py_XDECREF(builtins); - Py_XDECREF(super_args); - Py_XDECREF(super_type); - Py_XDECREF(super); - Py_XDECREF(func); - return result; - } diff --git a/doc/sphinx/source/thread_safety.rst b/doc/sphinx/source/thread_safety.rst index 89885e7..cee11e9 100644 --- a/doc/sphinx/source/thread_safety.rst +++ b/doc/sphinx/source/thread_safety.rst @@ -4,26 +4,66 @@ .. toctree:: :maxdepth: 2 -==================================== +.. index:: + single: Thread Safety + +******************************* Thread Safety +******************************* + +This chapter describes various issues when C extensions interact with Python threads [#f1]_. + ==================================== +When You Need a Lock +==================================== + +If your Extension is likely to be exposed to a multi-threaded environment then you need to think about thread safety. +I had this problem in a separate project which was a C++ `SkipList `_ +which could contain an ordered list of arbitrary Python objects. -If your Extension is likely to be exposed to a multi-threaded environment then you need to think about thread safety. I had this problem in a separate project which was a C++ `SkipList `_ which could contain an ordered list of arbitrary Python objects. The problem in a multi-threaded environment was that the following sequence of events could happen: +The problem in a multi-threaded environment that was sharing the same structure was that the following sequence of +events could happen: -* Thread A tries to insert a Python object into the SkipList. The C++ code searches for a place to insert it preserving the existing order. To do so it must call back into Python code for the user defined comparison function (using ``functools.total_ordering`` for example). -* At this point the Python interpreter is free to make a context switch allowing thread B to, say, remove an element from the SkipList. This removal may well invalidate C++ pointers held by thread A. +* Thread A tries to insert a Python object into the SkipList. + The C++ code searches for a place to insert it preserving the existing order. + To do so it must call back into Python code for the user defined comparison function + (using ``functools.total_ordering`` for example). +* At this point the Python interpreter is free to make a context switch allowing thread B to, say, remove an element + from the same SkipList. This removal may well invalidate C++ pointers held by thread A. * When the interpreter switches back to thread A it accesses an invalid pointer and a segfault happens. -The solution, of course, is to use a lock to prevent a context switch until A has completed its insertion, but how? I found the existing Python documentation misleading and I couldn't get it to work reliably, if at all. It was only when I stumbled upon the `source code `_ for the `bz module `_ that I realised there was a whole other, low level way of doing this, largely undocumented. +The solution is to use a lock to prevent a Python context switch until A has completed its insertion, but how? -.. note:: +I found the existing Python documentation misleading and I couldn't get it to work reliably, if at all. +It was only when I stumbled upon the `source code `_ +for the `bz module `_ that I realised there was a whole other, +low level way of doing this, largely undocumented. - Your Python may have been compiled without thread support in which case we don't have to concern ourselves with thread locking. We can discover this from the presence of the macro ``WITH_THREAD`` so all our thread support code is conditional on the definition of this macro. +Here is a version that concentrates on those essentials. +As an example, here is a subclass of a list that has a ``max()`` method that returns the maximum value in the list. +To do the comparison it must call +`PyObject_RichCompareBool `_ +to decide which of two objects is the maximum. +So during that call to ``max()`` the Python interpreter is free too switch to another thread that might alter the +list we are inspecting. +What we need to do is to block that thread with a lock so that can't happen. +Then once the result of ``max()`` is known we can relase that lock. +This class deliberately has ``sleep()`` calls to allow a thread switch to take place. + +The code (C and C++) is in ``src/cpy/Threads`` and the tests are in ``tests/unit/test_c_threads.py``. + +Lets walk through it. + +.. index:: + single: Thread Safety; Creating a Lock + +==================================== Coding up the Lock ----------------------------- +==================================== -First we need to include `pythread.h `_ as well as the usual includes: +First we need to include `pythread.h `_ +as well as the usual includes: .. code-block:: c :emphasize-lines: 4-6 @@ -35,60 +75,101 @@ First we need to include `pythread.h `_. Here is a fragment with the important lines highlighted: +Then we add a ``PyThread_type_lock`` (an opaque pointer) to the Python structure we are intending to protect. +Here is the object declaration: .. code-block:: c - :emphasize-lines: 4-6 typedef struct { - PyObject_HEAD - /* Other stuff here... */ + PyListObject list; #ifdef WITH_THREAD PyThread_type_lock lock; #endif - } SkipList; + } SubListObject; -Creating a class to Acquire and Release the Lock -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------------------------- +Initialising and Deallocating the Lock +-------------------------------------------------- -Now we add some code to acquire and release the lock. We can do this in a RAII fashion in C++ where the constructor blocks until the lock is acquired and the destructor releases the lock. The important lines are highlighted: +If you have a ``__new__`` method then set the lock pointer to ``NULL``. +The lock needs to be initialised only in the ``__init__`` method. +In the ``__init__`` method we allocate the lock by calling ``PyThread_allocate_lock()`` [#f2]_: .. code-block:: c - :emphasize-lines: 8-12,17 - #ifdef WITH_THREAD - /* A RAII wrapper around the PyThread_type_lock. */ - class AcquireLock { - public: - AcquireLock(SkipList *pSL) : _pSL(pSL) { - assert(_pSL); - assert(_pSL->lock); - if (! PyThread_acquire_lock(_pSL->lock, NOWAIT_LOCK)) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(_pSL->lock, WAIT_LOCK); - Py_END_ALLOW_THREADS - } + static int + SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) { + return -1; } - ~AcquireLock() { - assert(_pSL); - assert(_pSL->lock); - PyThread_release_lock(_pSL->lock); + #ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate thread lock."); + return -2; } - private: - SkipList *_pSL; - }; - #else - /* Make the class a NOP which should get optimised out. */ - class AcquireLock { - public: - AcquireLock(SkipList *) {} - }; #endif + return 0; + } -The code that acquires the lock is slightly clearer if the `Py_BEGIN_ALLOW_THREADS `_ and `Py_END_ALLOW_THREADS `_ macros are fully expanded [#f1]_: +When deallocating the object we should free the lock pointer with ``PyThread_free_lock`` [#f3]_: + +.. code-block:: c + + static void + SubList_dealloc(SubListObject *self) { + /* Deallocate other fields here. */ + #ifdef WITH_THREAD + if (self->lock) { + PyThread_free_lock(self->lock); + self->lock = NULL: + } + #endif + Py_TYPE(self)->tp_free((PyObject *)self); + } + +.. index:: + single: Thread Safety; Using a Lock + +==================================== +Using the Lock +==================================== + +So now our object has a lock but we need to acquire it and release it. + +------------------------------------- +From C Code +------------------------------------- + +It is useful to declare a couple of macros. +These are from the `bz module `_: + +.. code-block:: c + + #define ACQUIRE_LOCK(obj) do { \ + if (!PyThread_acquire_lock((obj)->lock, 0)) { \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((obj)->lock, 1); \ + Py_END_ALLOW_THREADS \ + } } while (0) + + #define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) + +The code that acquires the lock is slightly clearer if the +`Py_BEGIN_ALLOW_THREADS `_ +and `Py_END_ALLOW_THREADS `_ +macros are fully expanded: .. code-block:: c @@ -100,134 +181,319 @@ The code that acquires the lock is slightly clearer if the `Py_BEGIN_ALLOW_THREA PyEval_RestoreThread(_save); } } - -Initialising and Deallocating the Lock -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Before any critical section we need to use ``ACQUIRE_LOCK(self);`` (which blocks) then ``RELEASE_LOCK(self);`` +when done. +Failure to call ``RELEASE_LOCK(self);`` in any code path *will* lead to deadlocking. + +Example +------------------------------------- -Now we need to set the lock pointer to ``NULL`` in the ``_new`` function: +Here is an example of out sublist ``append()``. + +In the body of the function it makes a ``super()`` call and then introduces a ``sleep()`` which allows the +Python interpreter to switch threads (it should not because of the lock). .. code-block:: c - :linenos: - :emphasize-lines: 10-12 static PyObject * - SkipList_new(PyTypeObject *type, PyObject * /* args */, PyObject * /* kwargs */) { - SkipList *self = NULL; - - self = (SkipList *)type->tp_alloc(type, 0); - if (self != NULL) { - /* - * Initialise other struct SkipList fields... - */ - #ifdef WITH_THREAD - self->lock = NULL; - #endif - } - return (PyObject *)self; + SubList_append(SubListObject *self, PyObject *args) { + ACQUIRE_LOCK(self); + PyObject *result = call_super_name( + (PyObject *) self, "append", args, NULL + ); + // 0.25s delay to demonstrate holding on to the thread. + sleep_milliseconds(250L); + RELEASE_LOCK(self); + return result; } -In the ``__init__`` method we allocate the lock by calling ``PyThread_allocate_lock()`` [#f2]_ A lot of this code is specific to the SkipList but the lock allocation code is highlighted: +------------------------------------- +From C++ Code +------------------------------------- -.. code-block:: c - :linenos: - :emphasize-lines: 12-18 +We can make this a little smoother in C++ by creating a class that will lock and unlock. - static int - SkipList_init(SkipList *self, PyObject *args, PyObject *kwargs) { - int ret_val = -1; - PyObject *value_type = NULL; - PyObject *cmp_func = NULL; - static char *kwlist[] = { - (char *)"value_type", - (char *)"cmp_func", - NULL - }; - assert(self); - #ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - PyErr_SetString(PyExc_MemoryError, "Unable to allocate thread lock."); - goto except; - } - #endif - /* - * Much more stuff here... - */ - assert(! PyErr_Occurred()); - assert(self); - assert(self->pSl_void); - ret_val = 0; - goto finally; - except: - assert(PyErr_Occurred()); - Py_XDECREF(self); - ret_val = -1; - finally: - return ret_val; - } +Creating a class to Acquire and Release the Lock +---------------------------------------------------- -When deallocating the object we should free the lock pointer with ``PyThread_free_lock`` [#f3]_: +We can acquire and release the lock in a RAII fashion in C++ where the constructor blocks until the lock is acquired +and the destructor releases the lock. +This is a template class for generality. -.. code-block:: c - :linenos: - :emphasize-lines: 6-11 +The code is in ``src/cpy/Threads/cThreadLock.h`` + +.. code-block:: c++ + + #include + #include "structmember.h" - static void - SkipList_dealloc(SkipList *self) { - /* - * Deallocate other fields here... - */ #ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; + #include "pythread.h" + #endif + + #ifdef WITH_THREAD + /* A RAII wrapper around the PyThread_type_lock. */ + template + class AcquireLock { + public: + AcquireLock(T *pObject) : m_pObject(pObject) { + assert(m_pObject); + assert(m_pObject->lock); + Py_INCREF(m_pObject); + if (!PyThread_acquire_lock(m_pObject->lock, NOWAIT_LOCK)) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(m_pObject->lock, WAIT_LOCK); + Py_END_ALLOW_THREADS + } + } + ~AcquireLock() { + assert(m_pObject); + assert(m_pObject->lock); + PyThread_release_lock(m_pObject->lock); + Py_DECREF(m_pObject); } + private: + T *m_pObject; + }; + + #else + /* Make the class a NOP which should get optimised out. */ + template + class AcquireLock { + public: + AcquireLock(T *) {} + }; #endif - Py_TYPE(self)->tp_free((PyObject*)self); - } - } -Using the Lock -^^^^^^^^^^^^^^^^^^^^^^^^ +Using the AcquireLock class +---------------------------------------------------- -Before any critical code we create an ``AcquireLock`` object which blocks until we have the lock. Once the lock is obtained we can make any calls, including calls into the Python interpreter without preemption. The lock is automatically freed when we exit the code block: +Before any critical section we create an ``AcquireLock`` object which blocks until we have the lock. +Once the lock is obtained we can make any calls, including calls into the Python interpreter without preemption. +The lock is automatically freed when we exit the code block: .. code-block:: c - :linenos: - :emphasize-lines: 7,21 + /** append with a thread lock. */ static PyObject * - SkipList_insert(SkipList *self, PyObject *arg) { - assert(self && self->pSl_void); - /* Lots of stuff here... - */ - { - AcquireLock _lock(self); - /* We can make calls here, including calls back into the Python - * interpreter knowing that the interpreter will not preempt us. - */ - try { - self->pSl_object->insert(arg); - } catch (std::invalid_argument &err) { - // Thrown if PyObject_RichCompareBool returns -1 - // A TypeError should be set - if (! PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, err.what()); - } - return NULL; - } - /* Lock automatically released here. */ - } - /* More stuff here... - */ - Py_RETURN_NONE; + SubList_append(SubListObject *self, PyObject *args) { + AcquireLock local_lock((SubListObject *)self); + PyObject *result = call_super_name( + (PyObject *) self, "append", args, NULL + ); + // 0.25s delay to demonstrate holding on to the thread. + sleep_milliseconds(250L); + return result; } -And that is pretty much it. +.. index:: + single: Thread Safety; Examples + +==================================== +Example Code and Tests +==================================== + +The code (C and C++) is in ``src/cpy/Threads`` and the tests are in ``tests/unit/test_c_threads.py``. + +---------------------------------------------------- +Example Code +---------------------------------------------------- + +The example code is here: + +- ``C``: ``src/cpy/Threads/csublist.c`` +- ``C++``: ``src/cpy/Threads/cThreadLock.h`` and ``src/cpy/Threads/cppsublist.cpp``. + +``setup.py`` creates two extensions; ``cPyExtPatt.Threads.csublist`` (in C) and ``cPyExtPatt.Threads.cppsublist`` +(in C++): + +.. code-block:: python + + Extension(name=f"{PACKAGE_NAME}.Threads.csublist", + include_dirs=[ + '/usr/local/include', + 'src/cpy/Util', + "src/cpy/Threads", + ], + sources=[ + "src/cpy/Threads/csublist.c", + 'src/cpy/Util/py_call_super.c', + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(name=f"{PACKAGE_NAME}.Threads.cppsublist", + include_dirs=[ + '/usr/local/include', + 'src/cpy/Util', + "src/cpy/Threads", + ], + sources=[ + "src/cpy/Threads/cppsublist.cpp", + 'src/cpy/Util/py_call_super.c', + ], + language='c++11', + ), + + +The individual C and C++ modules can be accessed with: + +.. code-block:: python + + from cPyExtPatt.Threads import cppsublist + from cPyExtPatt.Threads import csublist + + +---------------------------------------------------- +Example Tests +---------------------------------------------------- + +The tests are in ``tests/unit/test_c_threads.py``. +Here are some examples: + +Tests in C +---------------------------------------------------- + +First create two function to call ``max()`` and ``append()``. +These functions print out their progress and which thread they are running in: + +.. code-block:: python + + def csublist_max(obj, count): + print( + f'sublist_max(): Thread name {threading.current_thread().name}', + flush=True + ) + for _i in range(count): + print( + f'sublist_max(): Thread name {threading.current_thread().name}' + f' Result: {obj.max()}', + flush=True + ) + time.sleep(0.25) + print( + f'sublist_max(): Thread name {threading.current_thread().name} DONE', + flush=True + ) + + + def csublist_append(obj, count): + print( + f'sublist_append(): Thread name {threading.current_thread().name}', + flush=True + ) + for _i in range(count): + print( + f'sublist_append(): Thread name {threading.current_thread().name}', + flush=True + ) + obj.append(len(obj)) + time.sleep(0.25) + print( + f'sublist_append(): Thread name {threading.current_thread().name} DONE', + flush=True + ) + + +Now a test that creates a single shared sub-list and four threads for each of the ``max()`` and ``append()`` +functions: + +.. code-block:: python + + def test_threaded_c(): + print() + print('test_threaded_c() START', flush=True) + obj = csublist.cSubList(range(128)) + threads = [] + for i in range(4): + threads.append( + threading.Thread( + name=f'sublist_max[{i:2d}]', + target=csublist_max, + args=(obj, 2), + ) + ) + threads.append( + threading.Thread( + name=f'sublist_append[{i:2d}]', + target=csublist_append, + args=(obj, 2), + ) + ) + for thread in threads: + thread.start() + print('Waiting for worker threads', flush=True) + main_thread = threading.current_thread() + for t in threading.enumerate(): + if t is not main_thread: + t.join() + print('Worker threads DONE', flush=True) + +Running this test gives this output, typically: + +.. code-block:: text + + test_threaded_c() START + sublist_max(): Thread name sublist_max[ 0] + sublist_append(): Thread name sublist_append[ 0] + sublist_append(): Thread name sublist_append[ 0] + sublist_max(): Thread name sublist_max[ 1] + sublist_append(): Thread name sublist_append[ 1] + sublist_max(): Thread name sublist_max[ 2] + sublist_max(): Thread name sublist_max[ 1] Result: 127 + sublist_append(): Thread name sublist_append[ 2] + sublist_max(): Thread name sublist_max[ 3] + sublist_append(): Thread name sublist_append[ 3] + Waiting for worker threads + sublist_append(): Thread name sublist_append[ 1] + sublist_max(): Thread name sublist_max[ 0] Result: 128 + sublist_max(): Thread name sublist_max[ 3] Result: 128 + sublist_append(): Thread name sublist_append[ 2] + sublist_append(): Thread name sublist_append[ 3] + sublist_append(): Thread name sublist_append[ 0] + sublist_max(): Thread name sublist_max[ 2] Result: 128 + sublist_append(): Thread name sublist_append[ 2] + sublist_append(): Thread name sublist_append[ 1] + sublist_max(): Thread name sublist_max[ 1] Result: 131 + sublist_max(): Thread name sublist_max[ 0] Result: 132 + sublist_append(): Thread name sublist_append[ 0] DONE + sublist_max(): Thread name sublist_max[ 1] DONE + sublist_max(): Thread name sublist_max[ 3] Result: 134 + sublist_append(): Thread name sublist_append[ 3] + sublist_append(): Thread name sublist_append[ 1] DONE + sublist_max(): Thread name sublist_max[ 2] Result: 134 + sublist_max(): Thread name sublist_max[ 0] DONE + sublist_append(): Thread name sublist_append[ 2] DONE + sublist_max(): Thread name sublist_max[ 3] DONE + sublist_append(): Thread name sublist_append[ 3] DONE + sublist_max(): Thread name sublist_max[ 2] DONE + Worker threads DONE + +Tests in C++ +---------------------------------------------------- + +A very similar example in C++ is in ``tests/unit/test_c_threads.py``. .. rubric:: Footnotes -.. [#f1] I won't pretend to understand all that is going on here, it does work however. -.. [#f2] What I don't understand is why putting this code in the ``SkipList_new`` function does not work, the lock does not get initialised and segfaults typically in ``_pthread_mutex_check_init``. The order has to be: set the lock pointer NULL in ``_new``, allocate it in ``_init``, free it in ``_dealloc``. -.. [#f3] A potiential weakness of this code is that we might be deallocating the lock *whilst the lock is acquired* which could lead to deadlock. This is very much implementation defined in ``pythreads`` and may vary from platform to platform. There is no obvious API in ``pythreads`` that allows us to determine if a lock is held so we can release it before deallocation. I notice that in the Python threading module (*Modules/_threadmodule.c*) there is an additional ``char`` field that acts as a flag to say when the lock is held so that the ``lock_dealloc()`` function in that module can release the lock before freeing the lock. \ No newline at end of file +.. [#f1] I don't cover 'pure' C threads (those that do not ``#include ``) here as they are not relevant. + If your C extension code creates/calls pure C threads this does not affect the CPython state for the *current* + thread. + Of course *that* C thread could embed another Python interpreter but that would have an entirely different state + than the current interpreter. + +.. [#f2] The order has to be: set the lock pointer NULL in ``_new``, allocate it in ``_init``, free it in ``_dealloc``. + If you don't do this then the lock does not get initialised and segfaults, typically in + ``_pthread_mutex_check_init``. + +.. [#f3] A potential weakness of this code is that we might be deallocating the lock *whilst the lock is acquired* + which could lead to deadlock. + + This is very much implementation defined in ``pythreads`` and may vary from platform to platform. + There is no obvious API in ``pythreads`` that allows us to determine if a lock is held so we can release it before + deallocation. + + I notice that in the Python threading module (*Modules/_threadmodule.c*) there is an additional ``char`` field that + acts as a flag to say when the lock is held so that the ``lock_dealloc()`` function in that module can release the + lock before freeing the lock. diff --git a/doc/sphinx/source/todo.rst b/doc/sphinx/source/todo.rst new file mode 100644 index 0000000..de5dece --- /dev/null +++ b/doc/sphinx/source/todo.rst @@ -0,0 +1,35 @@ +.. highlight:: python + :linenothreshold: 10 + +.. toctree:: + :maxdepth: 2 + +.. index:: + single: To Do + +==================================== +TODO +==================================== + +-------------------------------------------- +Existing TODOs +-------------------------------------------- + +.. todolist:: + +-------------------------------------------- +Work Estimate for v0.3.0 Release +-------------------------------------------- + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +New Chapter "Debugging Python with CLion" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Estimate: High [4] + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TOTAL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +4 points. diff --git a/doc/sphinx/source/watchers.rst b/doc/sphinx/source/watchers.rst new file mode 100644 index 0000000..993b472 --- /dev/null +++ b/doc/sphinx/source/watchers.rst @@ -0,0 +1,676 @@ +.. moduleauthor:: Paul Ross +.. sectionauthor:: Paul Ross + +.. highlight:: python + :linenothreshold: 30 + +.. toctree:: + :maxdepth: 3 + +.. + Links, mostly to the Python documentation. + Specific container links are just before the appropriate section. + +.. index:: + single: Watchers + +.. _chapter_watchers: + +.. index:: + single: Watchers + +====================================== +Watchers [Python 3.12+] +====================================== + +From Python 3.12 onwards *watchers* have been added [#]_. +This allows registering a callback function on specific ``dict``, ``type``, ``code`` or ``function`` object. +The callback is called with any event that occurs on the specific object. +This can be a powerful debugging technique. + +Here is an example of a dictionary watcher. + +.. index:: + pair: Watchers; Dictionary + +.. + Links to the Python dictionary documentation + +.. _PyDict_AddWatcher(): https://docs.python.org/3/c-api/dict.html#c.PyDict_AddWatcher +.. _PyDict_ClearWatcher(): https://docs.python.org/3/c-api/dict.html#c.PyDict_ClearWatcher +.. _PyDict_Watch(): https://docs.python.org/3/c-api/dict.html#c.PyDict_Watch +.. _PyDict_UnWatch(): https://docs.python.org/3/c-api/dict.html#c.PyDict_UnWatch +.. _PyDict_WatchEvent: https://docs.python.org/3/c-api/dict.html#c.PyDict_WatchEvent +.. _PyDict_WatchCallback(): https://docs.python.org/3/c-api/dict.html#c.PyDict_WatchCallback + +.. _chapter_watchers_dictionary: + +.. index:: + single: Watchers; Dictionary + +--------------------------- +Dictionary Watchers +--------------------------- + +Here is a context manager ``cWatchers.PyDictWatcher`` that wraps the low level CPython code with a watcher +that reports every dictionary operation to ``stdout``. +The code is in ``src/cpy/Watchers/watcher_example.py``. + +.. code-block:: python + :linenos: + + """Example of using watchers.""" + + from cPyExtPatt import cWatchers + + + def dict_watcher_demo() -> None: + print('dict_watcher_demo():') + d = {} # The dictionary we are going to watch. + with cWatchers.PyDictWatcher(d): + dd = {'age': 17, } + d.update(dd) # Generates event: PyDict_EVENT_CLONED + d['age'] = 42 # Generates event: PyDict_EVENT_MODIFIED + del d['age'] # Generates event: PyDict_EVENT_DELETED + d['name'] = 'Python' # Generates event: PyDict_EVENT_ADDED + d.clear() # Generates event: PyDict_EVENT_CLEARED + del d + + + if __name__ == '__main__': + dict_watcher_demo() + +And the output would be something like this, it reports the Python file, line number, function, event and +then detail about the arguments used to manipulate the dictionary: + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +.. + .. raw:: latex + + \begin{landscape} + +.. code-block:: text + + dict_watcher_demo(): + Dict @ 0x0x10fb53c00: watcher_example.py 11 dict_watcher_demo PyDict_EVENT_CLONED + Dict: {} + Key (dict): {'age': 17} + New value : NULL + Dict @ 0x0x10fb53c00: watcher_example.py 12 dict_watcher_demo PyDict_EVENT_MODIFIED + Dict: {'age': 17} + Key (str): age + New value (int): 42 + Dict @ 0x0x10fb53c00: watcher_example.py 13 dict_watcher_demo PyDict_EVENT_DELETED + Dict: {'age': 42} + Key (str): age + New value : NULL + Dict @ 0x0x10fb53c00: watcher_example.py 14 dict_watcher_demo PyDict_EVENT_ADDED + Dict: {} + Key (str): name + New value (str): Python + Dict @ 0x0x10fb53c00: watcher_example.py 15 dict_watcher_demo PyDict_EVENT_CLEARED + Dict: {'name': 'Python'} + Key : NULL + New value : NULL + +.. + .. raw:: latex + + \end{landscape} + +There are some obvious variations here: + +- Add some prefix to each watcher output line to discriminate it from the rest of stdout. +- Different outputs, such as JSON. + +But how does this watcher work? + +.. index:: + single: Watchers; Dictionary; Implementation + +Low Level C Implementation +-------------------------- + +We need some low level C code that interacts with the CPython watcher API. +First a header file that provides the interface to our dictionary watcher code. +This declares two functions: + +- ``dict_watcher_verbose_add()`` this adds a watcher to a dictionary and returns the watcher ID. +- ``dict_watcher_verbose_remove()`` this removes a watcher ID from a dictionary. + +The actual code is in ``src/cpy/Watchers/DictWatcher.h``. +It looks like this, note the Python version guard to ensure this only works with Python 3.12+: + +.. code-block:: c + + #define PPY_SSIZE_T_CLEAN + #include "Python.h" + + #if PY_VERSION_HEX < 0x030C0000 + + #error "Required version of Python is 3.12+ (PY_VERSION_HEX >= 0x030C0000)" + + #else + + int dict_watcher_verbose_add(PyObject *dict); + int dict_watcher_verbose_remove(int watcher_id, PyObject *dict); + + #endif // #if PY_VERSION_HEX >= 0x030C0000 + +So there are several moving parts in the implementation in ``src/cpy/Watchers/DictWatcher.c``. +First we have some general purpose functions that extract the file name, function name and line number from a Python +frame. +Note that the Python frame API changed in Python 3.11. + +First up, getting the Python file name: + +.. code-block:: c + + #include "DictWatcher.h" + + static const unsigned char * + get_python_file_name(PyFrameObject *frame) { + if (frame) { + // Python 3.11+ specific code. + #if PY_VERSION_HEX >= 0x030B0000 + /* See: + * https://docs.python.org/3.11/whatsnew/3.11.html#pyframeobject-3-11-hiding + */ + const unsigned char *file_name = PyUnicode_1BYTE_DATA( + PyFrame_GetCode(frame)->co_filename + ); + #else + const unsigned char *file_name = PyUnicode_1BYTE_DATA( + frame->f_code->co_filename + ); + #endif // #if PY_VERSION_HEX >= 0x030B0000 + return file_name; + } + return ""; + } + +Now, getting the Python function name: + +.. code-block:: c + + static const char * + get_python_function_name(PyFrameObject *frame) { + const char *func_name = NULL; + if (frame) { + // Python 3.11+ specific code. + #if PY_VERSION_HEX >= 0x030B0000 + /* See: + * https://docs.python.org/3.11/whatsnew/3.11.html#pyframeobject-3-11-hiding + */ + func_name = (const char *) PyUnicode_1BYTE_DATA( + PyFrame_GetCode(frame)->co_name + ); + #else + func_name = (const char *) PyUnicode_1BYTE_DATA(frame->f_code->co_name); + #endif // #if PY_VERSION_HEX >= 0x030B0000 + return func_name; + } + return ""; + } + +.. raw:: latex + + [Continued on the next page] + + \pagebreak + +Then, getting the Python line number: + +.. code-block:: c + + int get_python_line_number(PyFrameObject *frame) { + if (frame) { + return PyFrame_GetLineNumber(frame); + } + return 0; + } + +We bring these together to print a summary of the frame state to a file, such as ``stdout``: + +.. code-block:: c + + static void + write_frame_data_to_outfile(FILE *outfile, PyFrameObject *frame) { + if (frame) { + fprintf(outfile, + "%-80s %6d %-24s", + get_python_file_name(frame), + get_python_line_number(frame), + get_python_function_name(frame) + ); + } else { + fprintf(outfile, "No Python frame available."); + } + } + +Then there is a simple little helper function that returns a string based on the event type: + +.. code-block:: c + + static const char *watch_event_name(PyDict_WatchEvent event) { + switch (event) { + case PyDict_EVENT_ADDED: + return "PyDict_EVENT_ADDED"; + break; + case PyDict_EVENT_MODIFIED: + return "PyDict_EVENT_MODIFIED"; + break; + case PyDict_EVENT_DELETED: + return "PyDict_EVENT_DELETED"; + break; + case PyDict_EVENT_CLONED: + return "PyDict_EVENT_CLONED"; + break; + case PyDict_EVENT_CLEARED: + return "PyDict_EVENT_CLEARED"; + break; + case PyDict_EVENT_DEALLOCATED: + return "PyDict_EVENT_DEALLOCATED"; + break; + default: + Py_UNREACHABLE(); + break; + } + return "PyDict_EVENT_UNKNOWN"; + } + +Now we define the callback function that reports the dictionary event to ``stdout``. +This calles all the functionas above and uses the CPython API ``PyObject_Print`` to print the representation +of each object to ``stdout``. +This function has to respect NULL arguments: + +.. code-block:: c + + static int dict_watcher_verbose(PyDict_WatchEvent event, PyObject *dict, + PyObject *key, PyObject *new_value) { + fprintf(stdout, "Dict @ 0x%p: ", (void *)dict); + write_frame_data_to_outfile(stdout, PyEval_GetFrame()); + fprintf(stdout, " Event: %-24s", watch_event_name(event)); + fprintf(stdout, "\n"); + if (dict) { + fprintf(stdout, " Dict: "); + PyObject_Print(dict, stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, " Dict: NULL"); + } + fprintf(stdout, "\n"); + if (key) { + fprintf(stdout, " Key (%s): ", Py_TYPE(key)->tp_name); + PyObject_Print(key, stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, " Key : NULL"); + } + fprintf(stdout, "\n"); + if (new_value) { + fprintf(stdout, " New value (%s): ", Py_TYPE(new_value)->tp_name); + PyObject_Print(new_value, stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, " New value : NULL"); + } + fprintf(stdout, "\n"); + return 0; + } + +Finally we have the two implementations that register and unregister the callback using the low level Python C API. +This uses `PyDict_AddWatcher()`_ and `PyDict_Watch()`_. +The first registers the callback, returning the watcher ID (error handling code omitted): + +.. code-block:: c + + // Set watcher. + int dict_watcher_verbose_add(PyObject *dict) { + int watcher_id = PyDict_AddWatcher(&dict_watcher_verbose); + PyDict_Watch(watcher_id, dict); + return watcher_id; + } + +The second de-registers the callback, with the watcher ID and the dictionary in question +(error handling code omitted). +This uses `PyDict_Unwatch()`_ and `PyDict_ClearWatcher()`_. + +.. code-block:: c + + // Remove watcher. + int dict_watcher_verbose_remove(int watcher_id, PyObject *dict) { + PyDict_Unwatch(watcher_id, dict); + PyDict_ClearWatcher(watcher_id); + return 0; + } + +Exposing This to CPython +------------------------ + +Now we create a Python module ``cWatchers`` that exposes this low level C code to CPython. +This code is in ``src/cpy/Watchers/cWatchers.c``. + +First some module level CPython wrappers around our underlying C code: + +.. code-block:: c + + #define PPY_SSIZE_T_CLEAN + #include "Python.h" + + #include "DictWatcher.h" + + static PyObject * + py_dict_watcher_verbose_add(PyObject *Py_UNUSED(module), PyObject *arg) { + if (!PyDict_Check(arg)) { + PyErr_Format( + PyExc_TypeError, + "Argument must be a dict not type %s", + Py_TYPE(arg)->tp_name + ); + return NULL; + } + long watcher_id = dict_watcher_verbose_add(arg); + return Py_BuildValue("l", watcher_id); + } + + static PyObject * + py_dict_watcher_verbose_remove(PyObject *Py_UNUSED(module), PyObject *args) { + long watcher_id; + PyObject *dict = NULL; + + if (!PyArg_ParseTuple(args, "lO", &watcher_id, &dict)) { + return NULL; + } + + if (!PyDict_Check(dict)) { + PyErr_Format( + PyExc_TypeError, + "Argument must be a dict not type %s", + Py_TYPE(dict)->tp_name + ); + return NULL; + } + long result = dict_watcher_verbose_remove(watcher_id, dict); + return Py_BuildValue("l", result); + } + +Now create the table of module methods: + +.. code-block:: c + + static PyMethodDef module_methods[] = { + {"py_dict_watcher_verbose_add", + (PyCFunction) py_dict_watcher_verbose_add, + METH_O, + "Adds watcher to a dictionary. Returns the watcher ID." + }, + {"py_dict_watcher_verbose_remove", + (PyCFunction) py_dict_watcher_verbose_remove, + METH_VARARGS, + "Removes the watcher ID from the dictionary." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ + }; + +.. index:: + single: Watchers; Dictionary; Context Manager + +Creating the Context Manager +----------------------------- + +These are all fine but to be Pythonic it would be helpful to create a Context Manager +(see :ref:`chapter_context_manager`) in C. +The context manager holds a reference to the dictionary and the watcher ID. +Here is the definition which holds a watcher ID and a reference to the dictionary: + +.. code-block:: c + + typedef struct { + PyObject_HEAD + int watcher_id; + PyObject *dict; + } PyDictWatcher; + +Here is the creation code: + +.. code-block:: c + + static PyDictWatcher * + PyDictWatcher_new(PyObject *Py_UNUSED(arg)) { + PyDictWatcher *self; + self = PyObject_New(PyDictWatcher, &PyDictWatcher_Type); + if (self == NULL) { + return NULL; + } + self->watcher_id = -1; + self->dict = NULL; + return self; + } + + static PyObject * + PyDictWatcher_init(PyDictWatcher *self, PyObject *args) { + if (!PyArg_ParseTuple(args, "O", &self->dict)) { + return NULL; + } + if (!PyDict_Check(self->dict)) { + PyErr_Format( + PyExc_TypeError, + "Argument must be a dictionary not a %s", + Py_TYPE(self->dict)->tp_name + ); + return NULL; + } + Py_INCREF(self->dict); + return (PyObject *)self; + } + +The destruction code just decrements the reference count of the dictionary: + +.. code-block:: c + + static void + PyDictWatcher_dealloc(PyDictWatcher *self) { + Py_DECREF(self->dict); + PyObject_Del(self); + } + +Now the code that provides the context manager's ``__enter__`` and ``__exit__`` methods. +First the ``__enter__`` function, this uses the low level C function ``dict_watcher_verbose_add()`` to mark the +dictionary as watched and hold the watcher ID: + +.. code-block:: c + + static PyObject * + PyDictWatcher_enter(PyDictWatcher *self, PyObject *Py_UNUSED(args)) { + self->watcher_id = dict_watcher_verbose_add(self->dict); + Py_INCREF(self); + return (PyObject *)self; + } + +Now the ``__exit__`` function, this uses the low level C function ``dict_watcher_verbose_remove()`` to remove the +watcher from the dictionary: + +.. code-block:: c + + static PyObject * + PyDictWatcher_exit(PyDictWatcher *self, PyObject *Py_UNUSED(args)) { + int result = dict_watcher_verbose_remove(self->watcher_id, self->dict); + if (result) { + PyErr_Format( + PyExc_RuntimeError, + "dict_watcher_verbose_remove() returned %d", + result + ); + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; + } + +Now we define the context manager methods and type: + +.. code-block:: c + + static PyMethodDef PyDictWatcher_methods[] = { + {"__enter__", (PyCFunction) PyDictWatcher_enter, METH_VARARGS, + PyDoc_STR("__enter__() -> PyDictWatcher")}, + {"__exit__", (PyCFunction) PyDictWatcher_exit, METH_VARARGS, + PyDoc_STR("__exit__(exc_type, exc_value, exc_tb) -> bool")}, + {NULL, NULL, 0, NULL} /* sentinel */ + }; + + static PyTypeObject PyDictWatcher_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "cWatchers.PyDictWatcher", + .tp_basicsize = sizeof(PyDictWatcher), + .tp_dealloc = (destructor) PyDictWatcher_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = PyDictWatcher_methods, + .tp_new = (newfunc) PyDictWatcher_new, + .tp_init = (initproc) PyDictWatcher_init + }; + +.. index:: + single: Watchers; Dictionary; Module, Setup and Test + +Module, Setup and Test +----------------------------- + +Now we create the ``cWatchers`` module, + +.. code-block:: c + + static PyModuleDef cWatchers = { + PyModuleDef_HEAD_INIT, + .m_name = "cWatchers", + .m_doc = "Dictionary and type watchers.", + .m_size = -1, + .m_methods = module_methods, + }; + + PyMODINIT_FUNC PyInit_cWatchers(void) { + PyObject *m = PyModule_Create(&cWatchers); + if (!m) { + goto fail; + } + if (PyType_Ready(&PyDictWatcher_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "PyDictWatcher", (PyObject *) &PyDictWatcher_Type)) { + goto fail; + } + return m; + fail: + Py_XDECREF(m); + return NULL; + } + + +And then in ``setup.py`` we add the extension: + +.. code-block:: python + + Extension( + name=f"{PACKAGE_NAME}.cWatchers", + include_dirs=['src/cpy', ], + sources=[ + "src/cpy/Watchers/cWatchers.c", + "src/cpy/Watchers/DictWatcher.c", + "src/cpy/pyextpatt_util.c", + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + +And it can be used like this: + +.. code-block:: python + + from cPyExtPatt import cWatchers + + d = {} + with cWatchers.PyDictWatcher(d): + d['age'] = 42 + +.. + .. raw:: latex + + [Continued on the next page] + + \pagebreak + +And the result on ``stdout`` is something like: + +.. code-block:: text + + Dict @ 0x0x10fb53c00: watcher_example.py 12 dict_watcher_demo PyDict_EVENT_ADDED + Dict: {} + Key (str): age + New value (int): 42 + +.. index:: + single: Watchers; Dictionary; No Context Manager + +Without the Context Manager +--------------------------- + +If you are putting in some debugging code then a context manager might not be convenient. +``cWatchers`` provides two functions, ``py_dict_watcher_verbose_add()`` and +``py_dict_watcher_verbose_remove`` that achieve the same aim: + +.. code-block:: python + + from cPyExtPatt import cWatchers + + d = {} + watcher_id = cWatchers.py_dict_watcher_verbose_add(d) + d['age'] = 42 + cWatchers.py_dict_watcher_verbose_remove(watcher_id, d) + + +.. _PyType_AddWatcher(): https://docs.python.org/3/c-api/type.html#c.PyType_AddWatcher +.. _PyType_ClearWatcher(): https://docs.python.org/3/c-api/type.html#c.PyType_ClearWatcher +.. _PyType_Watch(): https://docs.python.org/3/c-api/type.html#c.PyType_Watch +.. _PyType_UnWatch(): https://docs.python.org/3/c-api/type.html#c.PyType_UnWatch +.. _PyType_WatchEvent: https://docs.python.org/3/c-api/type.html#c.PyType_WatchEvent +.. _PyType_WatchCallback(): https://docs.python.org/3/c-api/type.html#c.PyType_WatchCallback + +--------------------------- +Type Watchers +--------------------------- + +These allow a callback when a type is modified. + +This beyond the scope of this version of this document. + +More information can be found in https://docs.python.org/3/c-api/type.html + +--------------------------- +Function Watchers +--------------------------- + +These allow a callback when a function is created and destroyed. + +This beyond the scope of this version of this document. + +More information can be found in https://docs.python.org/3/c-api/function.html + +--------------------------- +Code Watchers +--------------------------- + +These allow a callback when code is created and destroyed. + +This beyond the scope of this version of this document. + +More information can be found in https://docs.python.org/3/c-api/code.html + +.. rubric:: Footnotes + +.. [#] This change was not done with any PEP that I can find. + Instead it was done during the ordinary pace of development. + For example this change has example tests: https://github.com/python/cpython/pull/31787/files + and is tracked with this issue: https://github.com/python/cpython/issues/91052 + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..121c092 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +Sphinx>=7.4 +psutil>=6.0 +pymemtrace>=0.2 +pytest>=8.3 +setuptools diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..078bacd --- /dev/null +++ b/setup.py @@ -0,0 +1,357 @@ +""" Usage: +python3 setup.py build + +Created on May 30, 2013 + +@author: paulross +""" +import os +import pathlib +import sys + +from setuptools import setup, Extension +import sysconfig + +here = pathlib.Path(__file__).parent.resolve() + +# Get the long description from the README file +long_description = ( + (here / 'README.rst').read_text(encoding='utf-8') + + '\n\n' + + (here / 'INSTALL.rst').read_text(encoding='utf-8') + + '\n\n' + + (here / 'HISTORY.rst').read_text(encoding='utf-8') +) + +licence = (here / 'LICENSE.txt').read_text(encoding='utf-8') + +DEBUG = True +# Generally I write code so that if DEBUG is defined as 0 then all optimisations +# are off and asserts are enabled. Typically run times of these builds are x2 to x10 +# release builds. +# If DEBUG > 0 then extra code paths are introduced such as checking the integrity of +# internal data structures. In this case the performance is by no means comparable +# with release builds. +DEBUG_LEVEL = 1 + +# Python stlib requirement: +LANGUAGE_STANDARD_C = "c99" +# Our level of C++ +LANGUAGE_STANDARD_CPP = "c++11" + +# Common flags for both release and debug builds. +# C +extra_compile_args_c = sysconfig.get_config_var('CFLAGS').split() +extra_compile_args_c += ["-std=%s" % LANGUAGE_STANDARD_C, "-Wall", "-Wextra"] +if DEBUG: + extra_compile_args_c += ["-g3", "-O0", "-DDEBUG=%s" % DEBUG_LEVEL, "-UNDEBUG"] +else: + extra_compile_args_c += ["-DNDEBUG", "-O3"] +# C++ +extra_compile_args_cpp = sysconfig.get_config_var('CFLAGS').split() +extra_compile_args_cpp += ["-std=%s" % LANGUAGE_STANDARD_CPP, "-Wall", "-Wextra"] +if DEBUG: + extra_compile_args_cpp += ["-g3", "-O0", "-DDEBUG=%s" % DEBUG_LEVEL, "-UNDEBUG"] +else: + extra_compile_args_cpp += ["-DNDEBUG", "-O3"] + +PYTHON_INCLUDE_DIRECTORIES = [ + sysconfig.get_paths()['include'], +] + +PACKAGE_NAME = 'cPyExtPatt' + +# Make directory cPyExtPatt/ and sub-directories such as Capsules/ +for dir_path in (os.path.join(os.path.dirname(__file__), 'cPyExtPatt'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'Capsules'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'cpp'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'SimpleExample'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'Iterators'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'SubClass'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'Threads'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'Logging'), + os.path.join(os.path.dirname(__file__), 'cPyExtPatt', 'RefCount'), + ): + if not os.path.exists(dir_path): + print(f'Making directory {dir_path}') + os.makedirs(dir_path) + pathlib.Path(os.path.join(dir_path, '__init__.py')).touch() + +# See: https://setuptools.pypa.io/en/latest/userguide/ext_modules.html +# language='c' or language='c++', +ext_modules = [ + Extension(f"{PACKAGE_NAME}.cExceptions", sources=['src/cpy/Exceptions/cExceptions.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cModuleGlobals", sources=['src/cpy/ModuleGlobals/cModuleGlobals.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cObject", sources=['src/cpy/Object/cObject.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cSeqObject", sources=['src/cpy/Object/cSeqObject.c', ], + include_dirs=['/usr/local/include', ], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cParseArgs", sources=['src/cpy/ParseArgs/cParseArgs.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + # Legacy code, see src/cpy/cParseArgsHelper.cpp for comments. + Extension(f"{PACKAGE_NAME}.cParseArgsHelper", sources=['src/cpy/ParseArgs/cParseArgsHelper.cpp', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_cpp, + language='c++11', + ), + Extension(f"{PACKAGE_NAME}.cPyRefs", sources=['src/cpy/RefCount/cPyRefs.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + # libraries = ['jpeg',], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cPickle", sources=['src/cpy/Pickle/cCustomPickle.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cFile", sources=[ + 'src/cpy/File/cFile.cpp', + 'src/cpy/File/PythonFileWrapper.cpp', + ], + include_dirs=['/usr/local/include', 'src/cpy/File', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_cpp, + language='c++11', + ), + Extension(f"{PACKAGE_NAME}.Capsules.spam", sources=['src/cpy/Capsules/spam.c', ], + include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.Capsules.spam_capsule", sources=['src/cpy/Capsules/spam_capsule.c', ], + include_dirs=['/usr/local/include', 'src/cpy/Capsules', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.Capsules.spam_client", sources=['src/cpy/Capsules/spam_client.c', ], + include_dirs=['/usr/local/include', 'src/cpy/Capsules', ], # os.path.join(os.getcwd(), 'include'),], + library_dirs=[os.getcwd(), ], # path to .a or .so file(s) + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.Capsules.datetimetz", + sources=[ + 'src/cpy/Capsules/datetimetz.c', + 'src/cpy/Util/py_call_super.c', + ], + include_dirs=[ + '/usr/local/include', + 'src/cpy/Capsules', + 'src/cpy/Util', + ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cpp.placement_new", + sources=['src/cpy/cpp/placement_new.cpp', ], + include_dirs=['/usr/local/include', 'src/cpy/cpp', ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_cpp, + language='c++11', + ), + Extension(f"{PACKAGE_NAME}.cpp.cUnicode", + sources=['src/cpy/cpp/cUnicode.cpp', ], + include_dirs=['/usr/local/include', ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_cpp, + language='c++11', + # undef_macros=undef_macros, + ), + Extension(f"{PACKAGE_NAME}.SimpleExample.cFibA", + sources=['src/cpy/SimpleExample/cFibA.c', ], + include_dirs=[], + library_dirs=[], + libraries=[], + # For best performance. + extra_compile_args=[ + '-Wall', '-Wextra', '-Werror', '-Wfatal-errors', '-Wpedantic', + '-Wno-unused-function', '-Wno-unused-parameter', + '-Qunused-arguments', '-std=c99', + '-UDEBUG', '-DNDEBUG', '-Ofast', '-g', + ], + language='c', + ), + Extension(f"{PACKAGE_NAME}.SimpleExample.cFibB", + sources=['src/cpy/SimpleExample/cFibB.c', ], + include_dirs=[], + library_dirs=[], + libraries=[], + # For best performance. + extra_compile_args=[ + '-Wall', '-Wextra', '-Werror', '-Wfatal-errors', '-Wpedantic', + '-Wno-unused-function', '-Wno-unused-parameter', + '-Qunused-arguments', '-std=c99', + '-UDEBUG', '-DNDEBUG', '-Ofast', '-g', + ], + language='c', + ), + # Extension(name=f"{PACKAGE_NAME}.Generators.gen_cpp", + # include_dirs=[], + # sources=["src/cpy/Generators/cGenerator.cpp", ], + # extra_compile_args=extra_compile_args_cpp, + # language='c++11', + # ), + Extension(name=f"{PACKAGE_NAME}.Iterators.cIterator", + include_dirs=[], + sources=["src/cpy/Iterators/cIterator.c", ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(name=f"{PACKAGE_NAME}.SubClass.sublist", + include_dirs=[ + '/usr/local/include', + 'src/cpy/Util', + ], + sources=[ + "src/cpy/SubClass/sublist.c", + 'src/cpy/Util/py_call_super.c', + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(name=f"{PACKAGE_NAME}.Threads.csublist", + include_dirs=[ + '/usr/local/include', + 'src/cpy/Util', + "src/cpy/Threads", + ], + sources=[ + "src/cpy/Threads/csublist.c", + 'src/cpy/Util/py_call_super.c', + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(name=f"{PACKAGE_NAME}.Threads.cppsublist", + include_dirs=[ + '/usr/local/include', + 'src/cpy/Util', + "src/cpy/Threads", + ], + sources=[ + "src/cpy/Threads/cppsublist.cpp", + 'src/cpy/Util/py_call_super.cpp', + ], + extra_compile_args=extra_compile_args_cpp, + language='c++11', + ), + Extension(name=f"{PACKAGE_NAME}.Logging.cLogging", + include_dirs=[], + sources=["src/cpy/Logging/cLogging.c", ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cRefCount", + include_dirs=[ + '/usr/local/include', + 'src/cpy', + ], + sources=[ + 'src/cpy/RefCount/cRefCount.c', + "src/cpy/pyextpatt_util.c", + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(f"{PACKAGE_NAME}.cCtxMgr", sources=['src/cpy/CtxMgr/cCtxMgr.c', ], + include_dirs=['/usr/local/include', ], + library_dirs=[os.getcwd(), ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + Extension(name=f"{PACKAGE_NAME}.cStructSequence", + include_dirs=[ + ], + sources=[ + "src/cpy/StructSequence/cStructSequence.c", + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), +] + +if sys.version_info.major >= 3 and sys.version_info.minor >= 12: + ext_modules.append( + Extension(name=f"{PACKAGE_NAME}.cWatchers", + include_dirs=[ + 'src/cpy', + 'src/cpy/Watchers', + ], + sources=[ + "src/cpy/Watchers/DictWatcher.c", + "src/cpy/pyextpatt_util.c", + "src/cpy/Watchers/cWatchers.c", + ], + extra_compile_args=extra_compile_args_c, + language='c', + ), + ) + +# For keywords see: https://setuptools.pypa.io/en/latest/references/keywords.html +setup( + name=PACKAGE_NAME, + version='0.3.0', + author='Paul Ross', + author_email='apaulross@gmail.com', + maintainer='Paul Ross', + maintainer_email='apaulross@gmail.com', + description='Python C Extension Patterns.', + url='https://github.com/paulross/PythonExtensionPatterns', + long_description=long_description, + long_description_content_type='text/x-rst', + platforms=['Mac OSX', 'POSIX', ], + packages=[PACKAGE_NAME, ], + # https://pypi.org/classifiers/ + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: C', + 'Programming Language :: C++', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Topic :: Software Development', + 'Topic :: Software Development :: Documentation', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + licence=licence, + # See: https://setuptools.pypa.io/en/latest/userguide/ext_modules.html + # language='c' or language='c++', + ext_modules=ext_modules +) diff --git a/src/FaultHandlerExample.py b/src/FaultHandlerExample.py index 4ccc8eb..885979d 100644 --- a/src/FaultHandlerExample.py +++ b/src/FaultHandlerExample.py @@ -1,7 +1,7 @@ import faulthandler faulthandler.enable() -import cPyRefs +from cPyExtPatt import cPyRefs -l = ['abc' * 200] -cPyRefs.popBAD(l) +a_list = ['abc' * 200] +cPyRefs.pop_and_print_BAD(a_list) diff --git a/src/build.sh b/src/build.sh deleted file mode 100755 index 7884ef6..0000000 --- a/src/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -rm -rf build/ -python3 setup.py build -cp build/lib.macosx-10.6-intel-3.3/*.so . -ls -l - diff --git a/src/cCanonical.c b/src/cCanonical.c index 411a67c..f66fce3 100644 --- a/src/cCanonical.c +++ b/src/cCanonical.c @@ -5,6 +5,8 @@ // Created by Paul Ross on 09/05/2014. // Copyright (c) 2014 Paul Ross. All rights reserved. // +// This is the Canonical CPython Function. +// This is example code for documentation and is not built. #include "Python.h" diff --git a/src/cExceptions.c b/src/cExceptions.c deleted file mode 100644 index f363eb7..0000000 --- a/src/cExceptions.c +++ /dev/null @@ -1,140 +0,0 @@ -// -// cExcep.c -// PythonExtensionPatterns -// -// Created by Paul Ross on 08/05/2014. -// Copyright (c) 2014 Paul Ross. All rights reserved. -// - -#include "Python.h" - -static PyObject *_raise_error(PyObject *module) { - - PyErr_SetString(PyExc_ValueError, "Ooops."); - assert(PyErr_Occurred()); - return NULL; -} - -static PyObject *_raise_error_formatted(PyObject *module) { - PyErr_Format(PyExc_ValueError, - "Can not read %d bytes when offset %d in byte length %d.", \ - 12, 25, 32 - ); - assert(PyErr_Occurred()); - return NULL; -} - -/* Illustrate returning NULL but not setting an exception. */ -static PyObject *_raise_error_bad(PyObject *module) { - PyErr_Clear(); - assert(! PyErr_Occurred()); - return NULL; -} - -/* Set and exception but fail to signal by returning non-NULL. */ -static PyObject *_raise_error_mixup(PyObject *module) { - PyErr_SetString(PyExc_ValueError, "ERROR: _raise_error_mixup()"); - assert(PyErr_Occurred()); - Py_RETURN_NONE; -} - -/* Test and exception, possibly set by another function. */ -static PyObject *_raise_error_mixup_test(PyObject *module) { - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; -} - -/* Shows that second PyErr_SetString() is ignored. */ -static PyObject *_raise_error_overwrite(PyObject *module) { - PyErr_SetString(PyExc_RuntimeError, "FORGOTTEN."); - PyErr_SetString(PyExc_ValueError, "ERROR: _raise_error_overwrite()"); - assert(PyErr_Occurred()); - return NULL; -} - -/* Specialise exceptions. */ -static PyObject *ExceptionBase; -static PyObject *SpecialisedError; - - -static PyMethodDef cExceptions_methods[] = { - {"raiseErr", (PyCFunction)_raise_error, METH_NOARGS, - "Raise a simple exception." - }, - {"raiseErrFmt", (PyCFunction)_raise_error_formatted, METH_NOARGS, - "Raise a formatted exception." - }, - {"raiseErrBad", (PyCFunction)_raise_error_bad, METH_NOARGS, - "Signal an exception by returning NULL but fail to set an exception." - }, - {"raiseErrMix", (PyCFunction)_raise_error_mixup, METH_NOARGS, - "Set an exception but fail to signal it but returning non-NULL." - }, - {"raiseErrTst", (PyCFunction)_raise_error_mixup_test, METH_NOARGS, - "Raise if an exception is set otherwise returns None." - }, - {"raiseErrOver", (PyCFunction)_raise_error_overwrite, METH_NOARGS, - "Example of overwriting exceptions, a RuntimeError is set then a ValueError. Only the latter is seen." - }, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -static PyModuleDef cExceptions_module = { - PyModuleDef_HEAD_INIT, - "cExceptions", - "Examples of raising exceptions.", - -1, - cExceptions_methods, - NULL, /* inquiry m_reload */ - NULL, /* traverseproc m_traverse */ - NULL, /* inquiry m_clear */ - NULL, /* freefunc m_free */ -}; - -PyMODINIT_FUNC -PyInit_cExceptions(void) -{ - PyObject* m = PyModule_Create(&cExceptions_module); - if (m == NULL) { - return NULL; - } - /* Initialise exceptions here. - * - * Firstly a base class exception that inherits from the builtin Exception. - * This is acheieved by passing NULL as the PyObject* as the third argument. - * - * PyErr_NewExceptionWithDoc returns a new reference. - */ - ExceptionBase = PyErr_NewExceptionWithDoc( - "cExceptions.ExceptionBase", /* char *name */ - "Base exception class for the noddy module.", /* char *doc */ - NULL, /* PyObject *base */ - NULL /* PyObject *dict */); - /* Error checking: this is oversimplified as it should decref - * anything created above such as m. - */ - if (! ExceptionBase) { - return NULL; - } else { - PyModule_AddObject(m, "ExceptionBase", ExceptionBase); - } - /* Now a sub-class exception that inherits from the base exception above. - * This is acheieved by passing non-NULL as the PyObject* as the third argument. - * - * PyErr_NewExceptionWithDoc returns a new reference. - */ - SpecialisedError = PyErr_NewExceptionWithDoc( - "cExceptions.SpecialsiedError", /* char *name */ - "Some specialised problem description here.", /* char *doc */ - ExceptionBase, /* PyObject *base */ - NULL /* PyObject *dict */); - if (! SpecialisedError) { - return NULL; - } else { - PyModule_AddObject(m, "SpecialisedError", SpecialisedError); - } - /* END: Initialise exceptions here. */ - return m; -} diff --git a/src/cObjmodule.c b/src/cObjmodule.c deleted file mode 100644 index 3a1501c..0000000 --- a/src/cObjmodule.c +++ /dev/null @@ -1,397 +0,0 @@ - -/* Use this file as a template to start implementing a module that - also declares object types. All occurrences of 'MyObj' should be changed - to something reasonable for your objects. After that, all other - occurrences of 'cObj' should be changed to something reasonable for your - module. If your module is named foo your sourcefile should be named - foomodule.c. - - You will probably want to delete all references to 'x_attr' and add - your own types of attributes instead. Maybe you want to name your - local variables other than 'self'. If your object type is needed in - other files, you'll have to create a file "foobarobject.h"; see - floatobject.h for an example. */ - -/* MyObj objects */ - -#include "Python.h" - -static PyObject *ErrorObject; - -typedef struct { - PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary */ -} MyObjObject; - -static PyTypeObject MyObj_Type; - -#define MyObjObject_Check(v) (Py_TYPE(v) == &MyObj_Type) - -static MyObjObject * -newMyObjObject(PyObject *arg) -{ - MyObjObject *self; - self = PyObject_New(MyObjObject, &MyObj_Type); - if (self == NULL) - return NULL; - self->x_attr = NULL; - return self; -} - -/* MyObj methods */ - -static void -MyObj_dealloc(MyObjObject *self) -{ - Py_XDECREF(self->x_attr); - PyObject_Del(self); -} - -static PyObject * -MyObj_demo(MyObjObject *self, PyObject *args) -{ - if (!PyArg_ParseTuple(args, ":demo")) - return NULL; - Py_INCREF(Py_None); - return Py_None; -} - -static PyMethodDef MyObj_methods[] = { - {"demo", (PyCFunction)MyObj_demo, METH_VARARGS, - PyDoc_STR("demo() -> None")}, - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -MyObj_getattro(MyObjObject *self, PyObject *name) -{ - if (self->x_attr != NULL) { - PyObject *v = PyDict_GetItem(self->x_attr, name); - if (v != NULL) { - Py_INCREF(v); - return v; - } - } - return PyObject_GenericGetAttr((PyObject *)self, name); -} - -static int -MyObj_setattr(MyObjObject *self, char *name, PyObject *v) -{ - if (self->x_attr == NULL) { - self->x_attr = PyDict_New(); - if (self->x_attr == NULL) - return -1; - } - if (v == NULL) { - int rv = PyDict_DelItemString(self->x_attr, name); - if (rv < 0) - PyErr_SetString(PyExc_AttributeError, - "delete non-existing MyObj attribute"); - return rv; - } - else - return PyDict_SetItemString(self->x_attr, name, v); -} - -static PyTypeObject MyObj_Type = { - /* The ob_type field must be initialized in the module init function - * to be portable to Windows without using C++. */ - PyVarObject_HEAD_INIT(NULL, 0) - "cObjmodule.MyObj", /*tp_name*/ - sizeof(MyObjObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)MyObj_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - (getattrfunc)0, /*tp_getattr*/ - (setattrfunc)MyObj_setattr, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - (getattrofunc)MyObj_getattro, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - MyObj_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ -}; -/* --------------------------------------------------------------------- */ - -/* Function of two integers returning integer */ - -PyDoc_STRVAR(cObj_foo_doc, -"foo(i,j)\n\ -\n\ -Return the sum of i and j."); - -static PyObject * -cObj_foo(PyObject *self, PyObject *args) -{ - long i, j; - long res; - if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) - return NULL; - res = i+j; /* cObjX Do something here */ - return PyLong_FromLong(res); -} - - -/* Function of no arguments returning new MyObj object */ - -static PyObject * -cObj_new(PyObject *self, PyObject *args) -{ - MyObjObject *rv; - - if (!PyArg_ParseTuple(args, ":new")) - return NULL; - rv = newMyObjObject(args); - if (rv == NULL) - return NULL; - return (PyObject *)rv; -} - -/* Example with subtle bug from extensions manual ("Thin Ice"). */ - -static PyObject * -cObj_bug(PyObject *self, PyObject *args) -{ - PyObject *list, *item; - - if (!PyArg_ParseTuple(args, "O:bug", &list)) - return NULL; - - item = PyList_GetItem(list, 0); - /* Py_INCREF(item); */ - PyList_SetItem(list, 1, PyLong_FromLong(0L)); - PyObject_Print(item, stdout, 0); - printf("\n"); - /* Py_DECREF(item); */ - - Py_INCREF(Py_None); - return Py_None; -} - -/* Test bad format character */ - -static PyObject * -cObj_roj(PyObject *self, PyObject *args) -{ - PyObject *a; - long b; - if (!PyArg_ParseTuple(args, "O#:roj", &a, &b)) - return NULL; - Py_INCREF(Py_None); - return Py_None; -} - - -/* ---------- */ - -static PyTypeObject Str_Type = { - /* The ob_type field must be initialized in the module init function - * to be portable to Windows without using C++. */ - PyVarObject_HEAD_INIT(NULL, 0) - "cObjmodule.Str", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /* see PyInit_cObj */ /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - -/* ---------- */ - -static PyObject * -null_richcompare(PyObject *self, PyObject *other, int op) -{ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; -} - -static PyTypeObject Null_Type = { - /* The ob_type field must be initialized in the module init function - * to be portable to Windows without using C++. */ - PyVarObject_HEAD_INIT(NULL, 0) - "cObjmodule.Null", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - null_richcompare, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /* see PyInit_cObj */ /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /* see PyInit_cObj */ /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ---------- */ - - -/* List of functions defined in the module */ - -static PyMethodDef cObj_methods[] = { - {"roj", cObj_roj, METH_VARARGS, - PyDoc_STR("roj(a,b) -> None")}, - {"foo", cObj_foo, METH_VARARGS, - cObj_foo_doc}, - {"new", cObj_new, METH_VARARGS, - PyDoc_STR("new() -> new cObj object")}, - {"bug", cObj_bug, METH_VARARGS, - PyDoc_STR("bug(o) -> None")}, - {NULL, NULL} /* sentinel */ -}; - -PyDoc_STRVAR(module_doc, -"This is a template module just for instruction."); - -/* Initialization function for the module (*must* be called PyInit_cObj) */ - - -static struct PyModuleDef cObjmodule = { - PyModuleDef_HEAD_INIT, - "cObj", - module_doc, - -1, - cObj_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit_cObj(void) -{ - PyObject *m = NULL; - - /* Due to cross platform compiler issues the slots must be filled - * here. It's required for portability to Windows without requiring - * C++. */ - Null_Type.tp_base = &PyBaseObject_Type; - Null_Type.tp_new = PyType_GenericNew; - Str_Type.tp_base = &PyUnicode_Type; - - /* Finalize the type object including setting type of the new type - * object; doing it here is required for portability, too. */ - if (PyType_Ready(&MyObj_Type) < 0) - goto fail; - //PyModule_AddObject(m, "Obj", (PyObject *)&MyObj_Type); - - /* Create the module and add the functions */ - m = PyModule_Create(&cObjmodule); - if (m == NULL) - goto fail; - - /* Add some symbolic constants to the module */ - if (ErrorObject == NULL) { - ErrorObject = PyErr_NewException("cObj.error", NULL, NULL); - if (ErrorObject == NULL) - goto fail; - } - Py_INCREF(ErrorObject); - PyModule_AddObject(m, "error", ErrorObject); - - /* Add Str */ - if (PyType_Ready(&Str_Type) < 0) - goto fail; - PyModule_AddObject(m, "Str", (PyObject *)&Str_Type); - - /* Add Null */ - if (PyType_Ready(&Null_Type) < 0) - goto fail; - PyModule_AddObject(m, "Null", (PyObject *)&Null_Type); - return m; - fail: - Py_XDECREF(m); - return NULL; -} diff --git a/src/cParseArgs.c b/src/cParseArgs.c deleted file mode 100644 index 87d1002..0000000 --- a/src/cParseArgs.c +++ /dev/null @@ -1,352 +0,0 @@ -// -// cParseArgs.c -// PythonExtensionPatterns -// -// Created by Paul Ross on 08/05/2014. -// Copyright (c) 2014 Paul Ross. All rights reserved. -// - -#include "Python.h" - -#include "time.h" - -/****************** Parsing arguments. ****************/ -static PyObject *_parse_no_args(PyObject *module) { - PyObject *ret = NULL; - - PyObject_Print(module, stdout, 0); - fprintf(stdout, "\n"); - - /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; -} - -static PyObject *_parse_one_arg(PyObject *module, PyObject *arg) { - PyObject *ret = NULL; - - PyObject_Print(module, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(arg, stdout, 0); - fprintf(stdout, "\n"); - /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; -} - -static PyObject *_parse_args(PyObject *module, PyObject *args) { - PyObject *ret = NULL; - PyObject *pyStr = NULL; - int arg1, arg2; - - PyObject_Print(module, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(args, stdout, 0); - fprintf(stdout, "\n"); - - if (!PyArg_ParseTuple(args, "Si|i", &pyStr, &arg1, &arg2)) { - goto except; - } - - /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; -} - -static PyObject *_parse_args_kwargs(PyObject *module, PyObject *args, - PyObject *kwargs) { - PyObject *ret = NULL; - PyObject *pyStr = NULL; - int arg2; - static char *kwlist[] = {"argOne", /* bytes object. */ - "argTwo", NULL}; - - PyObject_Print(module, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(args, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(kwargs, stdout, 0); - fprintf(stdout, "\n"); - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "S|i", kwlist, &pyStr, - &arg2)) { - goto except; - } - - /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; -} - -int _check_list_of_numbers(PyObject *lst, void *address) { - PyObject *item = NULL; - - if (!lst || !PyList_Check(lst)) { /* Note: PyList_Check allows sub-types. */ - return 0; - } - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(lst); ++i) { - item = PyList_GetItem(lst, i); - if (!(PyLong_CheckExact(item) || PyFloat_CheckExact(item) || - PyComplex_CheckExact(item))) { - PyErr_Format(PyExc_ValueError, "Item %d is not a number.", i); - return 0; - } - } - return 1; /* Success. */ -} - -/* Parse the args where we are expecting a single arg that must be a - * list of numbers. - */ -static PyObject *_parse_args_with_checking(PyObject *module, PyObject *args) { - PyObject *ret = NULL; - PyObject *pyObj = NULL; - - PyObject_Print(module, stdout, 0); - fprintf(stdout, "\n"); - PyObject_Print(args, stdout, 0); - fprintf(stdout, "\n"); - - if (!PyArg_ParseTuple(args, "O&", _check_list_of_numbers, &pyObj)) { - goto except; - } - - /* Your code here...*/ - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; -} - -/* Parse the args where we are simulating immutable defaults of a string and a - * tuple. - * This imitates the Python way of handling defaults. - */ -static PyObject *_parse_args_with_immutable_defaults(PyObject *module, - PyObject *args) { - PyObject *ret = NULL; - /* Pointers to the default arguments, initialised below. */ - static PyObject *pyObjDefaultArg_0; - static PyObject *pyObjDefaultArg_1; - /* These pointers are the ones we use in the body of the function, they - * either point at the supplied argument or the default (static) argument. - * We treat these as "borrowed" references and so indref and decref them - * appropriatly. - */ - PyObject *pyObjArg_0 = NULL; - PyObject *pyObjArg_1 = NULL; - - /* Set defaults for arguments. */ - if (!pyObjDefaultArg_0) { - pyObjDefaultArg_0 = PyUnicode_FromString("Hello world"); - if (!pyObjDefaultArg_0) { - PyErr_SetString(PyExc_RuntimeError, "Can not create string!"); - goto except; - } - } - if (!pyObjDefaultArg_1) { - pyObjDefaultArg_1 = PyTuple_New(2); - if (!pyObjDefaultArg_1) { - PyErr_SetString(PyExc_RuntimeError, "Can not create tuple!"); - goto except; - } - if (PyTuple_SetItem(pyObjDefaultArg_1, 0, PyLong_FromLong(42))) { - PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[0]!"); - goto except; - } - if (PyTuple_SetItem(pyObjDefaultArg_1, 1, PyUnicode_FromString("This"))) { - PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[1]!"); - goto except; - } - } - - if (!PyArg_ParseTuple(args, "|OO", &pyObjArg_0, &pyObjArg_1)) { - goto except; - } - /* If optional arguments absent then switch to defaults. */ - if (! pyObjArg_0) { - pyObjArg_0 = pyObjDefaultArg_0; - } - Py_INCREF(pyObjArg_0); - if (! pyObjArg_1) { - pyObjArg_1 = pyObjDefaultArg_1; - } - Py_INCREF(pyObjArg_1); - - fprintf(stdout, "pyObjArg0 was: "); - PyObject_Print(pyObjArg_0, stdout, 0); - fprintf(stdout, "\n"); - fprintf(stdout, "pyObjArg1 was: "); - PyObject_Print(pyObjArg_1, stdout, 0); - fprintf(stdout, "\n"); - - /* Your code here...*/ - - /* Mutate the arguments. */ - - fprintf(stdout, "pyObjArg0 now: "); - PyObject_Print(pyObjArg_0, stdout, 0); - fprintf(stdout, "\n"); - fprintf(stdout, "pyObjArg1 now: "); - PyObject_Print(pyObjArg_1, stdout, 0); - fprintf(stdout, "\n"); - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - assert(PyErr_Occurred()); - Py_XDECREF(ret); - ret = NULL; -finally: - Py_XDECREF(pyObjArg_0); - Py_XDECREF(pyObjArg_1); - return ret; -} - -/* Parse the args where we are simulating mutable defaults of a list and a dict. - * This imitates the Python way of handling defaults. - */ -static PyObject *_parse_args_with_mutable_defaults(PyObject *module, - PyObject *args) { - PyObject *ret = NULL; - /* Pointers to the default arguments, initialised below. */ - static PyObject *pyObjDefaultArg_0; - static PyObject *pyObjDefaultArg_1; - /* These pointers are the ones we use in the body of the function, they - * either point at the supplied argument or the default (static) argument. - * We treat these as "borrowed" references and so indref and decref them - * appropriatly. - */ - PyObject *pyObjArg_0 = NULL; - PyObject *pyObjArg_1 = NULL; - - /* Set defaults for arguments. */ - if (!pyObjDefaultArg_0) { - pyObjDefaultArg_0 = PyList_New(0); - } - if (!pyObjDefaultArg_1) { - pyObjDefaultArg_1 = PyDict_New(); - } - - if (!PyArg_ParseTuple(args, "|OO", &pyObjArg_0, &pyObjArg_1)) { - goto except; - } - /* If optional arguments absent then switch to defaults. */ - if (!pyObjArg_0) { - pyObjArg_0 = pyObjDefaultArg_0; - } - Py_INCREF(pyObjArg_0); - if (!pyObjArg_1) { - pyObjArg_1 = pyObjDefaultArg_1; - } - Py_INCREF(pyObjArg_1); - - fprintf(stdout, "pyObjArg0 was: "); - PyObject_Print(pyObjArg_0, stdout, 0); - fprintf(stdout, "\n"); - fprintf(stdout, "pyObjArg1 was: "); - PyObject_Print(pyObjArg_1, stdout, 0); - fprintf(stdout, "\n"); - - /* Your code here...*/ - - /* Mutate the arguments. */ - if (PyList_Append(pyObjArg_0, PyLong_FromLong(9))) { - PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); - goto except; - } - if (PyDict_SetItem(pyObjDefaultArg_1, - PyLong_FromLong(PyList_Size(pyObjArg_0)), - PyLong_FromLong(time(NULL)))) { - PyErr_SetString(PyExc_RuntimeError, "Can not append to dict!"); - goto except; - } - - fprintf(stdout, "pyObjArg0 now: "); - PyObject_Print(pyObjArg_0, stdout, 0); - fprintf(stdout, "\n"); - fprintf(stdout, "pyObjArg1 now: "); - PyObject_Print(pyObjArg_1, stdout, 0); - fprintf(stdout, "\n"); - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - assert(PyErr_Occurred()); - Py_XDECREF(ret); - ret = NULL; -finally: - Py_XDECREF(pyObjArg_0); - Py_XDECREF(pyObjArg_1); - return ret; -} - -static char _parse_args_kwargs_docstring[] = - "Some documentation for this function."; - -static PyMethodDef cParseArgs_methods[] = { - {"argsNone", (PyCFunction)_parse_no_args, METH_NOARGS, "No arguments."}, - {"argsOne", (PyCFunction)_parse_one_arg, METH_O, "One argument."}, - {"argsOnly", (PyCFunction)_parse_args, METH_VARARGS, "Reads args only."}, - {"argsKwargs", (PyCFunction)_parse_args_kwargs, - METH_VARARGS | METH_KEYWORDS, _parse_args_kwargs_docstring}, - {"argsListNums", (PyCFunction)_parse_args_with_checking, METH_VARARGS, - "Parsing an argument that must be a list of numbers."}, - {"argsImmutableDefault", (PyCFunction)_parse_args_with_immutable_defaults, - METH_VARARGS, "A function with mutable defaults."}, - {"argsMutableDefault", (PyCFunction)_parse_args_with_mutable_defaults, - METH_VARARGS, "A function with mutable defaults."}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -static PyModuleDef cParseArgs_module = { - PyModuleDef_HEAD_INIT, - "cParseArgs", - "Examples of parsing arguments in a Python 'C' extension.", - -1, - cParseArgs_methods, - NULL, /* inquiry m_reload */ - NULL, /* traverseproc m_traverse */ - NULL, /* inquiry m_clear */ - NULL, /* freefunc m_free */ -}; - -PyMODINIT_FUNC PyInit_cParseArgs(void) { - return PyModule_Create(&cParseArgs_module); -} -/****************** END: Parsing arguments. ****************/ diff --git a/src/cPyRefs.c b/src/cPyRefs.c deleted file mode 100644 index bf061ae..0000000 --- a/src/cPyRefs.c +++ /dev/null @@ -1,208 +0,0 @@ -// -// PyReferences.c -// PythonExtensionPatterns -// -// Created by Paul Ross on 07/05/2014. -// Copyright (c) 2014 Paul Ross. All rights reserved. -// -#include "Python.h" - -//#include - -/* - * 'New', 'stolen' and 'borrowed' references. - * These terms are used throughout the Python documentation, they refer to - * who is the real owner of the reference i.e. whose job it is to finally - * decref it (free it). - * - * This is all about programming by contract and each of reference types - * has a different contract. - */ - -/* New reference. - * This is object creation and it is your job to dispose of it. - * - * The analogy with 'C' is the reference has been malloc'd and must be free'd - * by you. - */ -static PyObject *subtract_long(long a, long b) { - PyObject *pA, *pB, *r; - - pA = PyLong_FromLong(a); /* pA: New reference. */ - pB = PyLong_FromLong(b); /* pB: New reference. */ - r = PyNumber_Subtract(pA, pB); /* r: New reference. */ - Py_DECREF(pA); /* My responsibility to decref. */ - Py_DECREF(pB); /* My responsibility to decref. */ - return r; /* Callers responsibility to decref. */ -} - -static PyObject *subtract_two_longs(PyObject *pModule) { - return subtract_long(421, 17); -} - -static PyObject *access_after_free(PyObject *pModule) { - PyObject *pA = PyLong_FromLong(1024L); - Py_DECREF(pA); - PyObject_Print(pA, stdout, 0); - Py_RETURN_NONE; -} - - -/* Stolen reference. - * This is object creation but where another object takes responsibility - * for decref'ing (freeing) the object. - * These are quite rare; typical examples are object insertion into tuples - * lists, dicts etc. - * - * The analogy with C would be malloc'ing some memory, populating it and - * inserting that pointer into a linked list where the linked list promises - * to free the memory when that item in the list is removed. - */ -static PyObject *make_tuple(PyObject *pModule) { - PyObject *r; - PyObject *v; - - r = PyTuple_New(3); /* New reference. */ - fprintf(stdout, "Ref count new: %zd\n", r->ob_refcnt); - v = PyLong_FromLong(1L); /* New reference. */ - /* PyTuple_SetItem steals the new reference v. */ - PyTuple_SetItem(r, 0, v); - /* This is fine. */ - v = PyLong_FromLong(2L); - PyTuple_SetItem(r, 1, v); - /* More common pattern. */ - PyTuple_SetItem(r, 2, PyUnicode_FromString("three")); - return r; /* Callers responsibility to decref. */ -} - -void handle_list(PyObject *pList) { - while (PyList_Size(pList) > 0) { - PySequence_DelItem(pList, PyList_Size(pList) - 1); - } -} - -/* 'Borrowed' reference this is when reading from an object, you get back a - * reference to something that the object still owns _and_ the container - * can dispose of at _any_ time. - * The problem is that you might want that reference for longer. - */ -static PyObject *pop_and_print_BAD(PyObject *pModule, PyObject *pList) { - PyObject *pLast; - - pLast = PyList_GetItem(pList, PyList_Size(pList) - 1); - fprintf(stdout, "Ref count was: %zd\n", pLast->ob_refcnt); - /* ... stuff here ... */ - handle_list(pList); - /* ... more stuff here ... */ - fprintf(stdout, "Ref count now: %zd\n", pLast->ob_refcnt); - PyObject_Print(pLast, stdout, 0); /* Boom. */ - fprintf(stdout, "\n"); - Py_RETURN_NONE; -} - -static PyObject *pop_and_print_OK(PyObject *pModule, PyObject *pList) { - PyObject *pLast; - - pLast = PyList_GetItem(pList, PyList_Size(pList) - 1); - fprintf(stdout, "Ref count was: %zd\n", pLast->ob_refcnt); - Py_INCREF(pLast); - fprintf(stdout, "Ref count now: %zd\n", pLast->ob_refcnt); - /* ... stuff here ... */ - handle_list(pList); - /* ... more stuff here ... */ - PyObject_Print(pLast, stdout, 0); - fprintf(stdout, "\n"); - Py_DECREF(pLast); - fprintf(stdout, "Ref count fin: %zd\n", pLast->ob_refcnt); - - Py_RETURN_NONE; -} - -/* Just increfs a PyObject. */ -static PyObject *incref(PyObject *pModule, PyObject *pObj) { - fprintf(stdout, "incref(): Ref count was: %zd\n", pObj->ob_refcnt); - Py_INCREF(pObj); - fprintf(stdout, "incref(): Ref count now: %zd\n", pObj->ob_refcnt); - Py_RETURN_NONE; -} - -/* Just decrefs a PyObject. */ -static PyObject *decref(PyObject *pModule, PyObject *pObj) { - fprintf(stdout, "decref(): Ref count was: %zd\n", pObj->ob_refcnt); - Py_DECREF(pObj); - fprintf(stdout, "decref(): Ref count now: %zd\n", pObj->ob_refcnt); - Py_RETURN_NONE; -} - -/* This leaks new references. - */ -static PyObject *leak_new_reference(PyObject *pModule, - PyObject *args, PyObject *kwargs) { - PyObject *ret = NULL; - int value, count; - static char *kwlist[] = {"value", "count", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ii", kwlist, &value, - &count)) { - goto except; - } - fprintf(stdout, "loose_new_reference: value=%d count=%d\n", value, count); - for (int i = 0; i < count; ++i) { - PyLong_FromLong(value); /* New reference, leaked. */ - } - - Py_INCREF(Py_None); - ret = Py_None; - goto finally; -except: - Py_XDECREF(ret); - ret = NULL; -finally: - fprintf(stdout, "loose_new_reference: DONE\n"); - return ret; -} - - -static PyMethodDef cPyRefs_methods[] = { - {"newRef", (PyCFunction)subtract_two_longs, METH_NOARGS, - "Returns a new long by subtracting two longs in Python." - }, - {"stealRef", (PyCFunction)make_tuple, METH_NOARGS, - "Creates a tuple by stealing new references." - }, - {"popBAD", (PyCFunction)pop_and_print_BAD, METH_O, - "Borrowed refs, might segfault." - }, - {"popOK", (PyCFunction)pop_and_print_OK, METH_O, - "Borrowed refs, should not segfault." - }, - {"incref", (PyCFunction)incref, METH_O, - "incref a PyObject." - }, - {"decref", (PyCFunction)decref, METH_O, - "decref a PyObject." - }, - {"leakNewRefs", (PyCFunction)leak_new_reference, - METH_VARARGS | METH_KEYWORDS, "Leaks new references to longs."}, - {"afterFree", (PyCFunction)access_after_free, - METH_NOARGS, "Example of access after decrement reference."}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -static PyModuleDef cPyRefs_module = { - PyModuleDef_HEAD_INIT, - "cPyRefs", - "Examples of reference types in a 'C' extension.", - -1, - cPyRefs_methods, - NULL, /* inquiry m_reload */ - NULL, /* traverseproc m_traverse */ - NULL, /* inquiry m_clear */ - NULL, /* freefunc m_free */ -}; - -PyMODINIT_FUNC -PyInit_cPyRefs(void) -{ - return PyModule_Create(&cPyRefs_module); -} diff --git a/src/cpy/Capsules/custom_capsule.c b/src/cpy/Capsules/custom_capsule.c new file mode 100644 index 0000000..514d761 --- /dev/null +++ b/src/cpy/Capsules/custom_capsule.c @@ -0,0 +1,219 @@ +// +// Created by Paul Ross on 18/06/2021. +// +// This takes the Python 3.9 custom3 example and shows how to us Capsules with it. + +#define PY_SSIZE_T_CLEAN +#include +#include "structmember.h" +#define CUSTOM_CAPSULE +#include "custom_capsule.h" + +typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; +} CustomObject; + +static void +Custom_dealloc(CustomObject *self) { + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +Custom_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { + CustomObject *self; + self = (CustomObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->first = PyUnicode_FromString(""); + if (self->first == NULL) { + Py_DECREF(self); + return NULL; + } + self->last = PyUnicode_FromString(""); + if (self->last == NULL) { + Py_DECREF(self); + return NULL; + } + self->number = 0; + } + return (PyObject *) self; +} + +static int +Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_DECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_DECREF(tmp); + } + return 0; +} + +static PyMemberDef Custom_members[] = { + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL, 0, 0, 0, NULL} /* Sentinel */ +}; + +static PyObject * +Custom_getfirst(CustomObject *self, void *Py_UNUSED(closure)) { + Py_INCREF(self->first); + return self->first; +} + +static int +Custom_setfirst(CustomObject *self, PyObject *value, void *Py_UNUSED(closure)) { + PyObject *tmp; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The first attribute value must be a string"); + return -1; + } + tmp = self->first; + Py_INCREF(value); + self->first = value; + Py_DECREF(tmp); + return 0; +} + +static PyObject * +Custom_getlast(CustomObject *self, void *Py_UNUSED(closure)) { + Py_INCREF(self->last); + return self->last; +} + +static int +Custom_setlast(CustomObject *self, PyObject *value, void *Py_UNUSED(closure)) { + PyObject *tmp; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The last attribute value must be a string"); + return -1; + } + tmp = self->last; + Py_INCREF(value); + self->last = value; + Py_DECREF(tmp); + return 0; +} + +static PyGetSetDef Custom_getsetters[] = { + {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + "first name", NULL}, + {"last", (getter) Custom_getlast, (setter) Custom_setlast, + "last name", NULL}, + {NULL, NULL, NULL, NULL, NULL} /* Sentinel */ +}; + +static PyObject * +Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) { + return PyUnicode_FromFormat("%S %S", self->first, self->last); +} + +static PyMethodDef Custom_methods[] = { + {"name", (PyCFunction) Custom_name, METH_NOARGS, + "Return the name, combining the first and last name" + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "custom3.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = Custom_new, + .tp_init = (initproc) Custom_init, + .tp_dealloc = (destructor) Custom_dealloc, + .tp_members = Custom_members, + .tp_methods = Custom_methods, + .tp_getset = Custom_getsetters, +}; + +static PyModuleDef custommodule = { + PyModuleDef_HEAD_INIT, + .m_name = "custom_capsule", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +/* C API. Clients get at this via PyDateTime_IMPORT, defined in + * datetime.h. + */ +static PyCustom_CAPI CAPI = { + &CustomType +// &PyDateTime_DateTimeType, +// &PyDateTime_TimeType, +// &PyDateTime_DeltaType, +// &PyDateTime_TZInfoType, +// NULL, // PyDatetime_TimeZone_UTC not initialized yet +// new_date_ex, +// new_datetime_ex, +// new_time_ex, +// new_delta_ex, +// new_timezone, +// datetime_fromtimestamp, +// datetime_date_fromtimestamp_capi, +// new_datetime_ex2, +// new_time_ex2 +}; + + +PyMODINIT_FUNC +PyInit_custom_capsule(void) +{ + PyObject *m; + if (PyType_Ready(&CustomType) < 0) + return NULL; + + m = PyModule_Create(&custommodule); + if (m == NULL) + return NULL; + + Py_INCREF(&CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + + PyObject *c_api_object = PyCapsule_New(&CAPI, PyCustom_CAPSULE_NAME, NULL); + if (c_api_object == NULL) { + return NULL; + } + if (PyModule_AddObject(m, "CAPI", c_api_object) < 0) { + Py_XDECREF(c_api_object); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/Capsules/custom_capsule.h b/src/cpy/Capsules/custom_capsule.h new file mode 100644 index 0000000..9d9722b --- /dev/null +++ b/src/cpy/Capsules/custom_capsule.h @@ -0,0 +1,69 @@ +// +// Created by Paul Ross on 18/06/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_CUSTOM_CAPSULE_H +#define PYTHONEXTENSIONSBASIC_CUSTOM_CAPSULE_H +#ifdef __cplusplus +extern "C" { +#endif + +/* Define structure for C API. */ +typedef struct { + /* type objects */ + PyTypeObject *CustomType; +// PyTypeObject *DateTimeType; +// PyTypeObject *TimeType; +// PyTypeObject *DeltaType; +// PyTypeObject *TZInfoType; +// +// /* singletons */ +// PyObject *TimeZone_UTC; +// +// /* constructors */ +// PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*); +// PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, +// PyObject*, PyTypeObject*); +// PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*); +// PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*); +// PyObject *(*TimeZone_FromTimeZone)(PyObject *offset, PyObject *name); +// +// /* constructors for the DB API */ +// PyObject *(*DateTime_FromTimestamp)(PyObject*, PyObject*, PyObject*); +// PyObject *(*Date_FromTimestamp)(PyObject*, PyObject*); +// +// /* PEP 495 constructors */ +// PyObject *(*DateTime_FromDateAndTimeAndFold)(int, int, int, int, int, int, int, +// PyObject*, int, PyTypeObject*); +// PyObject *(*Time_FromTimeAndFold)(int, int, int, int, PyObject*, int, PyTypeObject*); +// +} PyCustom_CAPI; + +#define PyCustom_CAPSULE_NAME "custom3_capsule.CAPI" + +#ifdef CUSTOM_CAPSULE +/* Code that is used for a standard build of custom such as done by setup.py. */ + +#else +/* Code that provides a C API to custom. */ +static void **PyCustom_API; + +#define PySpam_System \ + (*(PySpam_System_RETURN (*)PySpam_System_PROTO) PySpam_API[PySpam_System_NUM]) + +/* Return -1 on error, 0 on success. + * PyCapsule_Import will set an exception if there's an error. + */ +static int +import_custom(void) +{ + PyCustom_API = (void **)PyCapsule_Import("custom._C_API", 0); + return (PyCustom_API != NULL) ? 0 : -1; +} + +#endif + +#ifdef __cplusplus +} +#endif +#endif //PYTHONEXTENSIONSBASIC_CUSTOM_CAPSULE_H diff --git a/src/cpy/Capsules/custom_use.c b/src/cpy/Capsules/custom_use.c new file mode 100644 index 0000000..c21f42e --- /dev/null +++ b/src/cpy/Capsules/custom_use.c @@ -0,0 +1,22 @@ +// +// Created by Paul Ross on 19/06/2021. +// +#define PY_SSIZE_T_CLEAN +#include + +#include "custom_capsule.h" + + +//PyMODINIT_FUNC +//PyInit_customuse(void) +//{ +// PyObject *m; +// +// m = PyModule_Create(&clientmodule); +// if (m == NULL) +// return NULL; +// if (import_custom() < 0) +// return NULL; +// /* additional initialization can happen here */ +// return m; +//} diff --git a/src/cpy/Capsules/datetimetz.c b/src/cpy/Capsules/datetimetz.c new file mode 100644 index 0000000..c7b46ef --- /dev/null +++ b/src/cpy/Capsules/datetimetz.c @@ -0,0 +1,173 @@ +// +// Created by Paul Ross on 13/07/2024. +// +// Implements a datatimetz subclass of datetime that always has a timezone. +// This is an example of using Capsules and the datetime Capsule API. + +#define PY_SSIZE_T_CLEAN + +#include +#include "datetime.h" +#include "py_call_super.h" + +#define FPRINTF_DEBUG 0 + +/* From /Library/Frameworks/Python.framework/Versions/3.13/include/python3.13/object.h + * These were introduced in Python 3.10: https://docs.python.org/3/c-api/structures.html#c.Py_IsNone + * */ +#if PY_MINOR_VERSION < 10 +// Test if the 'x' object is the 'y' object, the same as "x is y" in Python. +PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); +#define Py_Is(x, y) ((x) == (y)) + +// Test if an object is the None singleton, the same as "x is None" in Python. +PyAPI_FUNC(int) Py_IsNone(PyObject *x); +#define Py_IsNone(x) Py_Is((x), Py_None) + +//PyAPI_FUNC(int) _PyDateTime_HAS_TZINFO(PyObject *datetime) { +// if (datetime->tzinfo == NULL) { +// return -1; +// } else if (Py_IsNone(datetime->tzinfo)) { +// PyErr_SetString(PyExc_TypeError, "No time zone provided."); +// return -2; +// } +// return 0; +//} +#endif + +typedef struct { + PyDateTime_DateTime datetime; +} DateTimeTZ; + +/** + * Does a version dependent check to see if a datatimetz has a tzinfo. + * If not, sets an error and returns NULL. + */ +static DateTimeTZ * +raise_if_no_tzinfo(DateTimeTZ *self) { + // Raise if no TZ. +#if PY_MINOR_VERSION >= 10 + if (!_PyDateTime_HAS_TZINFO(&self->datetime)) { + PyErr_SetString(PyExc_TypeError, "No time zone provided."); + Py_DECREF(self); + self = NULL; + } +#else // PY_MINOR_VERSION < 10 + if (self->datetime.tzinfo == NULL) { + PyErr_SetString(PyExc_TypeError, "No time zone provided."); + Py_DECREF(self); + self = NULL; + } else if (Py_IsNone(self->datetime.tzinfo)) { + PyErr_SetString(PyExc_TypeError, "No time zone provided."); + Py_DECREF(self); + self = NULL; + } +#endif // PY_MINOR_VERSION + return self; +} + +static PyObject * +DateTimeTZ_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { +#if FPRINTF_DEBUG + fprintf(stdout, "DateTimeTZ_new() type:\n"); + PyObject_Print((PyObject *)type, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + fprintf(stdout, "DateTimeTZ_new() args:\n"); + PyObject_Print(args, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + fprintf(stdout, "DateTimeTZ_new() kwds:\n"); + PyObject_Print(kwds, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); +#endif + DateTimeTZ *self = (DateTimeTZ *) PyDateTimeAPI->DateTimeType->tp_new(type, args, kwds); + if (self) { +#if FPRINTF_DEBUG + fprintf(stdout, "DateTimeTZ_new() self:\n"); + PyObject_Print((PyObject *)self, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + fprintf(stdout, "DateTimeTZ_new() &self->datetime:\n"); + PyObject_Print((PyObject*)(&self->datetime), stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); +#if PY_MINOR_VERSION >= 10 + fprintf(stdout, "DateTimeTZ_new() _PyDateTime_HAS_TZINFO(&self->datetime): %d\n", _PyDateTime_HAS_TZINFO(&self->datetime)); +#else // PY_MINOR_VERSION >= 10 + fprintf(stdout, "DateTimeTZ_new() self->datetime.tzinfo:\n"); + if ((void *)&self->datetime != NULL && (&self->datetime)->tzinfo) { +// fprintf(stdout, "tzinfo %p %s\n", (void *)(&self->datetime)->tzinfo, Py_TYPE((&self->datetime)->tzinfo)->tp_name); + PyObject_Print((PyObject *) ((&self->datetime)->tzinfo), stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, "No tzinfo\n"); + } + fprintf(stdout, "\n"); +#endif // PY_MINOR_VERSION < 10 +#endif + self = raise_if_no_tzinfo(self); + } + return (PyObject *) self; +} + +static PyObject * +DateTimeTZ_replace(PyObject *self, PyObject *args, PyObject *kwargs) { +// PyObject_Print(self, stdout, 0); + PyObject *result = call_super_name(self, "replace", args, kwargs); +// PyObject_Print(self, stdout, 0); + if (result) { + result = (PyObject *) raise_if_no_tzinfo((DateTimeTZ *) result); + } + return result; +} + +static PyMethodDef DateTimeTZ_methods[] = { + { + "replace", + (PyCFunction) DateTimeTZ_replace, + METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Return datetime with new specified fields.") + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject DatetimeTZType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "datetimetz.datetimetz", + .tp_doc = "A datetime that requires a time zone.", + .tp_basicsize = sizeof(DateTimeTZ), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = DateTimeTZ_new, + .tp_methods = DateTimeTZ_methods, +}; + +static PyModuleDef datetimetzmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "datetimetz", + .m_doc = ( + "Module that contains a datetimetz," + "a datetime.datetime with a mandatory time zone." + ), + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_datetimetz(void) { + PyObject *m = PyModule_Create(&datetimetzmodule); + if (m == NULL) { + return NULL; + } + // datetime.datetime_CAPI + PyDateTime_IMPORT; + if (!PyDateTimeAPI) { + Py_DECREF(m); + return NULL; + } + // Set inheritance. + DatetimeTZType.tp_base = PyDateTimeAPI->DateTimeType; + if (PyType_Ready(&DatetimeTZType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&DatetimeTZType); + PyModule_AddObject(m, "datetimetz", (PyObject *) &DatetimeTZType); + /* additional initialization can happen here */ + return m; +} diff --git a/src/cpy/Capsules/spam.c b/src/cpy/Capsules/spam.c new file mode 100644 index 0000000..0f0d3be --- /dev/null +++ b/src/cpy/Capsules/spam.c @@ -0,0 +1,46 @@ +// +// Created by Paul Ross on 13/07/2024. +// +// Implements: https://docs.python.org/3/extending/extending.html#extending-simpleexample +// Excludes specific exception. +// Lightly edited. + +#define PY_SSIZE_T_CLEAN +#include + +static PyObject * +spam_system(PyObject *Py_UNUSED(self), PyObject *args) { + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +static PyMethodDef SpamMethods[] = { + /* ... */ + {"system", spam_system, METH_VARARGS, + "Execute a shell command."}, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef spammodule = { + PyModuleDef_HEAD_INIT, + "spam", /* name of module */ + PyDoc_STR("Documentation for the spam module"), /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods, + NULL, + NULL, + NULL, + NULL, +}; + +PyMODINIT_FUNC +PyInit_spam(void) { + return PyModule_Create(&spammodule); +} diff --git a/src/cpy/Capsules/spam_capsule.c b/src/cpy/Capsules/spam_capsule.c new file mode 100644 index 0000000..6f244a6 --- /dev/null +++ b/src/cpy/Capsules/spam_capsule.c @@ -0,0 +1,73 @@ +// +// Created by Paul Ross on 13/07/2024. +// +// Implements: https://docs.python.org/3/extending/extending.html#extending-simpleexample +// as a capsule: https://docs.python.org/3/extending/extending.html#providing-a-c-api-for-an-extension-module +// Includes specific exception. +// Lightly edited. + +#define PY_SSIZE_T_CLEAN + +#include + +#define SPAM_CAPSULE + +#include "spam_capsule.h" + +static int +PySpam_System(const char *command) { + return system(command); +} + +static PyObject * +spam_system(PyObject *Py_UNUSED(self), PyObject *args) { + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = PySpam_System(command); + return PyLong_FromLong(sts); +} + +static PyMethodDef SpamMethods[] = { + /* ... */ + {"system", spam_system, METH_VARARGS, + "Execute a shell command."}, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef spammodule = { + PyModuleDef_HEAD_INIT, + "spam_capsule", /* name of module */ + PyDoc_STR("Documentation for the spam module"), /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods, + NULL, NULL, NULL, NULL, +}; + +PyMODINIT_FUNC +PyInit_spam_capsule(void) { + PyObject *m; + static void *PySpam_API[PySpam_API_pointers]; + PyObject *c_api_object; + + m = PyModule_Create(&spammodule); + if (m == NULL) + return NULL; + + /* Initialize the C API pointer array */ + PySpam_API[PySpam_System_NUM] = (void *) PySpam_System; + + /* Create a Capsule containing the API pointer array's address */ + c_api_object = PyCapsule_New((void *) PySpam_API, "cPyExtPatt.Capsules.spam_capsule._C_API", NULL); + + if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { + Py_XDECREF(c_api_object); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/Capsules/spam_capsule.h b/src/cpy/Capsules/spam_capsule.h new file mode 100644 index 0000000..6ed50f6 --- /dev/null +++ b/src/cpy/Capsules/spam_capsule.h @@ -0,0 +1,43 @@ +#ifndef Py_SPAM_CAPSULE_H +#define Py_SPAM_CAPSULE_H +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Header file for spammodule */ + +/* C API functions */ +#define PySpam_System_NUM 0 +#define PySpam_System_RETURN int +#define PySpam_System_PROTO (const char *command) + +/* Total number of C API pointers */ +#define PySpam_API_pointers 1 + +#ifdef SPAM_CAPSULE + +/* This section is used when compiling spam_capsule.c */ +static PySpam_System_RETURN PySpam_System PySpam_System_PROTO; + +#else +/* This section is used in modules that use spam_capsule's API */ +static void **PySpam_API; + +#define PySpam_System \ + (*(PySpam_System_RETURN (*)PySpam_System_PROTO) PySpam_API[PySpam_System_NUM]) + +/* Return -1 on error, 0 on success. + * PyCapsule_Import will set an exception if there's an error. + */ +static int +import_spam_capsule(void) { + PySpam_API = (void **)PyCapsule_Import("cPyExtPatt.Capsules.spam_capsule._C_API", 0); + return (PySpam_API != NULL) ? 0 : -1; +} +#endif +#ifdef __cplusplus +} +#endif +#endif /* !defined(Py_SPAM_CAPSULE_H) */ diff --git a/src/cpy/Capsules/spam_client.c b/src/cpy/Capsules/spam_client.c new file mode 100644 index 0000000..8b44995 --- /dev/null +++ b/src/cpy/Capsules/spam_client.c @@ -0,0 +1,54 @@ +// +// Created by Paul Ross on 13/07/2024. +// +// Implements: https://docs.python.org/3/extending/extending.html#extending-simpleexample +// but using a capsule exported by spam_capsule.h/.c +// Excludes specific exception. +// Lightly edited. + +#define PY_SSIZE_T_CLEAN + +#include +#include "spam_capsule.h" + +static PyObject * +spam_system(PyObject *Py_UNUSED(self), PyObject *args) { + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = PySpam_System(command); + return PyLong_FromLong(sts); +} + +static PyMethodDef SpamMethods[] = { + /* ... */ + {"system", spam_system, METH_VARARGS, + "Execute a shell command."}, + /* ... */ + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef spam_clientmodule = { + PyModuleDef_HEAD_INIT, + "spam_client", /* name of module */ + PyDoc_STR("Documentation for the spam module"), /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods, + NULL, NULL, NULL, NULL, +}; + +PyMODINIT_FUNC +PyInit_spam_client(void) { + PyObject *m; + + m = PyModule_Create(&spam_clientmodule); + if (m == NULL) + return NULL; + if (import_spam_capsule() < 0) + return NULL; + /* additional initialization can happen here */ + return m; +} diff --git a/src/cpy/Containers/DebugContainers.c b/src/cpy/Containers/DebugContainers.c new file mode 100644 index 0000000..d8c5942 --- /dev/null +++ b/src/cpy/Containers/DebugContainers.c @@ -0,0 +1,3001 @@ +// +// Created by Paul Ross on 15/12/2024. +// +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +#include "DebugContainers.h" + +#pragma mark - Tuples + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SetItem. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyTuple_SetItem_steals(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyTuple_SetItem(container, 0, value)) { + assert(0); + } + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SET_ITEM. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyTuple_SET_ITEM_steals(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyTuple_SET_ITEM(container, 0, value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SetItem on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyTuple_SetItem_steals_replace(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + int result = PyTuple_SetItem(container, 0, value_0); + assert(result == 0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + /* Preserve the value_1 so that we can see Py_DECREF(container) decrements it. */ + Py_INCREF(value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 2); + + /* This will decrement the ref count of value_0 leaving it with a reference count of 1. + * Whilst preserving the reference count of value_1 of 2. */ + PyTuple_SetItem(container, 0, value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 2); + + /* Check that value_0 has a ref count of 1. */ + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + assert(get_item == value_1); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + /* Clean up. */ + Py_DECREF(container); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + Py_DECREF(value_0); + Py_DECREF(value_1); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SET_ITEM on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyTuple_SET_ITEM_steals_replace(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyTuple_SET_ITEM(container, 0, value_0); + + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + + /* This will overwrite value_0 leaving it with a reference count of 1.*/ + PyTuple_SET_ITEM(container, 0, value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + assert(get_item == value_1); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + + /* This is demonstrated as leaked. */ + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); +} + +void dbg_PyTuple_SetItem_replace_with_same(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + int result = PyTuple_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* Increment the reference count to track the bad behaviour. */ + Py_INCREF(value); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + /* This will decrement the reference count of value as it is the previous value. + * That will free the current value and set garbage in the tuple. */ + result = PyTuple_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + /* This is only alive because of Py_INCREF(value); above. */ + assert(ref_count == 1); + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + /* Increment the reference count from 1 so we can see it go back to 1 on Py_DECREF(container);. */ + Py_INCREF(value); + Py_DECREF(container); + /* Clean up. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyTuple_SET_ITEM_replace_with_same(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + PyTuple_SET_ITEM(container, 0, value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* Second PyTuple_SET_ITEM(). */ + /* This will NOT decrement the reference count of value as it is the previous value. */ + PyTuple_SET_ITEM(container, 0, value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + /* Increment the reference count from 1 so we can see it go back to 1 on Py_DECREF(container);. */ + Py_INCREF(value); + Py_DECREF(container); + /* Clean up. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyTuple_SetItem(). + */ +void dbg_PyTuple_SetIem_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyTuple_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyTuple_SET_ITEM(). + */ +void dbg_PyTuple_SET_ITEM_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyTuple_SET_ITEM(container, 0, NULL); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyTuple_SetItem() then setting it to a value. + */ +void dbg_PyTuple_SetIem_NULL_SetItem(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyTuple_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + PyTuple_SetItem(container, 0, value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + Py_DECREF(container); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyTuple_SET_ITEM() then setting it to a value. + */ +void dbg_PyTuple_SET_ITEM_NULL_SET_ITEM(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyTuple_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyTuple_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + PyTuple_SET_ITEM(container, 0, value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + Py_DECREF(container); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks PyTuple_SetItem when the container is not a tuple. + * This decrements the value reference count. + */ +void dbg_PyTuple_SetItem_fails_not_a_tuple(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* We want to to hold onto this as PyTuple_SetItem() will decref it. */ + Py_INCREF(value); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + int result = PyTuple_SetItem(container, 0, value); + assert(result == -1); + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + + /* Yes, has been decremented on failure. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks PyTuple_SetItem when the container is not a tuple. + * This decrements the value reference count. + */ +void dbg_PyTuple_SetItem_fails_out_of_range(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* We want to to hold onto this as PyTuple_SetItem() will decref it. */ + Py_INCREF(value); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + int result = PyTuple_SetItem(container, 1, value); + assert(result == -1); + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + + /* Yes, has been decremented on failure. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores PyTuple_Pack(n, ...). + */ +void dbg_PyTuple_PyTuple_Pack(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + + PyObject *container = PyTuple_Pack(2, value_a, value_b); + + assert(Py_REFCNT(value_a) == 2); + assert(Py_REFCNT(value_b) == 2); + + Py_DECREF(container); + + /* Leaks: */ + assert(Py_REFCNT(value_a) == 1); + assert(Py_REFCNT(value_b) == 1); + + Py_DECREF(value_a); + Py_DECREF(value_b); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores Py_BuildValue("(O)", ...). + */ +void dbg_PyTuple_Py_BuildValue(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *container = Py_BuildValue("(O)", value); + + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +#pragma mark - Lists + +/** + * A function that checks whether a tuple steals a reference when using PyList_SetItem. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyList_SetItem_steals(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyList_SetItem(container, 0, value)) { + assert(0); + } + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a tuple steals a reference when using PyList_SET_ITEM. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyList_SET_ITEM_steals(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyList_SET_ITEM(container, 0, value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a tuple steals a reference when using PyList_SetItem on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * This DOES leak an existing value contrary to the Python documentation. + */ +void dbg_PyList_SetItem_steals_replace(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + int result = PyList_SetItem(container, 0, value_0); + assert(result == 0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + /* Preserve the value_1 so that we can see Py_DECREF(container) decrements it. */ + Py_INCREF(value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 2); + + /* This will decrement the ref count of value_0 leaving it with a reference count of 1.*/ + PyList_SetItem(container, 0, value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 2); + + /* Check that value_0 has a ref count of 1. */ + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + assert(get_item == value_1); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + /* Clean up. */ + Py_DECREF(container); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + Py_DECREF(value_0); + Py_DECREF(value_1); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a tuple steals a reference when using PyList_SET_ITEM on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyList_SET_ITEM_steals_replace(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyList_SET_ITEM(container, 0, value_0); + + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + + /* This will overwrite value_0 leaving it with a reference count of 1.*/ + PyList_SET_ITEM(container, 0, value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 1); + PyObject *get_item = PyList_GET_ITEM(container, 0); + assert(get_item == value_1); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + Py_DECREF(container); + + /* This is demonstrated as leaked. */ + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_SetItem_replace_with_same(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + int result = PyList_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* Increment the reference count to track the bad behaviour. */ + Py_INCREF(value); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + /* This will decrement the reference count of value as it is the previous value. + * That will free the current value and set garbage in the tuple. */ + result = PyList_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + /* This is only alive because of Py_INCREF(value); above. */ + assert(ref_count == 1); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + /* Increment the reference count from 1 so we can see it go back to 1 on Py_DECREF(container);. */ + Py_INCREF(value); + Py_DECREF(container); + /* Clean up. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_SET_ITEM_replace_with_same(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + PyList_SET_ITEM(container, 0, value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* Second PyList_SET_ITEM(). */ + /* This will NOT decrement the reference count of value as it is the previous value. */ + PyList_SET_ITEM(container, 0, value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + /* Increment the reference count from 1 so we can see it go back to 1 on Py_DECREF(container);. */ + Py_INCREF(value); + Py_DECREF(container); + /* Clean up. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyList_SetItem(). + */ +void dbg_PyList_SetIem_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyList_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyList_SET_ITEM(). + */ +void dbg_PyList_SET_ITEM_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyList_SET_ITEM(container, 0, NULL); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyList_SetItem() then setting it to a value. + */ +void dbg_PyList_SetIem_NULL_SetItem(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyList_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + PyList_SetItem(container, 0, value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + Py_DECREF(container); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores setting an item in a tuple to NULL with PyList_SET_ITEM() then setting it to a value. + */ +void dbg_PyList_SET_ITEM_NULL_SET_ITEM(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + PyObject *container = PyList_New(1); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + PyList_SetItem(container, 0, NULL); + assert(!PyErr_Occurred()); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + PyList_SET_ITEM(container, 0, value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + Py_DECREF(container); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 1); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks PyList_SetItem when the container is not a tuple. + * This decrements the value reference count. + */ +void dbg_PyList_SetItem_fails_not_a_tuple(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* We want to to hold onto this as PyList_SetItem() will decref it. */ + Py_INCREF(value); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + int result = PyList_SetItem(container, 0, value); + assert(result == -1); + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + + /* Yes, has been decremented on failure. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks PyList_SetItem when the container is not a tuple. + * This decrements the value reference count. + */ +void dbg_PyList_SetItem_fails_out_of_range(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* We want to to hold onto this as PyList_SetItem() will decref it. */ + Py_INCREF(value); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + int result = PyList_SetItem(container, 1, value); + assert(result == -1); + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + + /* Yes, has been decremented on failure. */ + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +/** + * A function that checks whether a list increments a reference when using PyList_Append. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + */ +void dbg_PyList_Append(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(0); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyList_Append(container, value)) { + assert(0); + } + // PyList_Append increments. + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + /* Need this. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Append_fails_not_a_list(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + assert(!PyErr_Occurred()); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + int result = PyList_Append(container, value); + assert(result); + + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Append_fails_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(0); + assert(container); + assert(!PyErr_Occurred()); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + int result = PyList_Append(container, NULL); + assert(result); + + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Insert(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(0); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(PyList_GET_SIZE(container) == 0); + if (PyList_Insert(container, 0L, value)) { + assert(0); + } + assert(PyList_GET_SIZE(container) == 1); + // PyList_Append increments. + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item = PyList_GET_ITEM(container, 0); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + /* Need this. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Insert_Is_Truncated(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(0); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyList_Insert(container, 4L, value)) { + assert(0); + } + // PyList_Insert increments. + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item; + // PyList_Insert at 4 actually inserts at 0. + assert(PyList_GET_SIZE(container) == 1L); + get_item = PyList_GET_ITEM(container, 0L); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + /* Need this. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Insert_Negative_Index(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(0); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyList_Insert(container, -1L, value)) { + assert(0); + } + // PyList_Insert increments. + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item; + // PyList_Insert at -1 actually inserts at 0. + assert(PyList_GET_SIZE(container) == 1L); + get_item = PyList_GET_ITEM(container, 0L); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + /* Need this. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Insert_fails_not_a_list(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + assert(!PyErr_Occurred()); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + int result = PyList_Insert(container, 1L, value); + assert(result); + + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyList_Insert_fails_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + assert(container); + assert(!PyErr_Occurred()); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + int result = PyList_Insert(container, 1L, NULL); + assert(result); + + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +/** + * Function that explores Py_BuildValue("(O)", ...). + */ +void dbg_PyList_Py_BuildValue(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *container = Py_BuildValue("[O]", value); + + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +#pragma mark - Dictionaries - setters + +void dbg_PyDict_SetItem_increments(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_a); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value_a)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_a); + assert(ref_count == 2); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value_a); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + /* Now replace the value using the same key. */ + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_b); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value_b)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_a); + assert(ref_count == 1); + ref_count = Py_REFCNT(value_b); + assert(ref_count == 2); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value_b); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + // Replace with existing key/value_b. Reference counts should remain the same. + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_b); + assert(ref_count == 2); + if (PyDict_SetItem(container, key, value_b)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_b); + assert(ref_count == 2); + + Py_DECREF(container); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value_b); + assert(ref_count == 1); + + Py_DECREF(key); + Py_DECREF(value_a); + Py_DECREF(value_b); + + assert(!PyErr_Occurred()); +} + +#if ACCEPT_SIGSEGV + +void dbg_PyDict_SetItem_NULL_key(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + PyObject *container = PyDict_New(); + assert(container); + + PyObject *key = NULL; + PyObject *value = new_unique_string(__FUNCTION__, NULL); + // Segfault + PyDict_SetItem(container, key, value); +} + +void dbg_PyDict_SetItem_NULL_value(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + PyObject *container = PyDict_New(); + assert(container); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *value = NULL; + // Segfault + PyDict_SetItem(container, key, value); +} + +#endif // ACCEPT_SIGSEGV + +void dbg_PyDict_SetItem_fails_not_a_dict(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(0); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + int result = PyDict_SetItem(container, key, value); + if (result) { + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); /* Clears the error. */ + } else { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_SetItem_fails_not_hashable(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = PyList_New(0); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + int result = PyDict_SetItem(container, key, value); + if (result) { + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); /* Clears the error. */ + } else { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_SetDefault_default_unused(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + /* Now check PyDict_SetDefault() which does not use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 1); + + get_item = PyDict_SetDefault(container, key, value_default); + if (! get_item) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 1); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + assert(get_item == value); + + Py_DECREF(container); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value_default); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_SetDefault_default_used(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + /* Do not do this so the default is invoked. + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + */ + + /* Now check PyDict_SetDefault() which *does* use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 1); + + get_item = PyDict_SetDefault(container, key, value_default); + if (! get_item) { + assert(0); + } + assert(PyDict_Size(container) == 1); + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 2); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + assert(get_item == value_default); + + Py_DECREF(container); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value_default); + + assert(!PyErr_Occurred()); +} + +#pragma mark - Dictionaries [Python3.13] + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +// PyDict_SetDefaultRef +// int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) +// https://docs.python.org/3/c-api/dict.html#c.PyDict_SetDefaultRef +void dbg_PyDict_SetDefaultRef_default_unused(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item = PyDict_GetItem(container, key); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + /* Now check PyDict_SetDefault() which does not use the default. */ + PyObject *default_value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(default_value); + assert(ref_count == 1); + + /* From https://docs.python.org/3/c-api/dict.html#c.PyDict_SetDefaultRef lightly edited. + * + * Inserts default_value into the dictionary p with a key of key if the key is not already + * present in the dictionary. + * If result is not NULL, then *result is set to a strong reference to either default_value, + * if the key was not present, or the existing value, if key was already present in the dictionary. + * Returns: + * 1 if the key was present and default_value was not inserted. + * 0 if the key was not * present and default_value was inserted. + * -1 on failure, sets an exception, and sets *result to NULL. + * + * For clarity: if you have a strong reference to default_value before calling this function, + * then after it returns, you hold a strong reference to both default_value and *result (if it’s not NULL). + * These may refer to the same object: in that case you hold two separate references to it. + */ + PyObject *result = NULL; + int return_value = PyDict_SetDefaultRef(container, key, default_value, &result); + if (return_value != 1) { + assert(0); + } + + assert(result == value); + + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 3); + ref_count = Py_REFCNT(default_value); + assert(ref_count == 1); + ref_count = Py_REFCNT(result); + assert(ref_count == 3); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 3); + assert(get_item == value); + + Py_DECREF(container); + + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + ref_count = Py_REFCNT(default_value); + assert(ref_count == 1); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value); + Py_DECREF(default_value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_SetDefaultRef_default_used(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 1); + + /* From https://docs.python.org/3/c-api/dict.html#c.PyDict_SetDefaultRef lightly edited. + * + * Inserts default_value into the dictionary p with a key of key if the key is not already + * present in the dictionary. + * If result is not NULL, then *result is set to a strong reference to either default_value, + * if the key was not present, or the existing value, if key was already present in the dictionary. + * Returns: + * 1 if the key was present and default_value was not inserted. + * 0 if the key was not * present and default_value was inserted. + * -1 on failure, sets an exception, and sets *result to NULL. + * + * For clarity: if you have a strong reference to default_value before calling this function, + * then after it returns, you hold a strong reference to both default_value and *result (if it’s not NULL). + * These may refer to the same object: in that case you hold two separate references to it. + */ + PyObject *result = NULL; + int return_value = PyDict_SetDefaultRef(container, key, value_default, &result); + if (return_value != 0) { + assert(0); + } + + assert(result == value_default); + + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 3); + ref_count = Py_REFCNT(result); + assert(ref_count == 3); + + Py_DECREF(container); + + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 2); + ref_count = Py_REFCNT(result); + assert(ref_count == 2); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value_default); + Py_DECREF(value_default); + + assert(!PyErr_Occurred()); +} + +/* + * This explores using PyDict_SetDefaultRef when result is a live Python object. + * The previous version of result is abandoned. + */ +void dbg_PyDict_SetDefaultRef_default_unused_result_non_null(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item = PyDict_GetItem(container, key); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + /* Now check PyDict_SetDefault() which does not use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 1); + + PyObject *result_live = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(result_live); + assert(ref_count == 1); + + PyObject *result = result_live; + int return_value = PyDict_SetDefaultRef(container, key, value_default, &result); + if (return_value != 1) { + assert(0); + } + + assert(result != result_live); + assert(result == value); + + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 3); + ref_count = Py_REFCNT(value_default); + assert(ref_count == 1); + ref_count = Py_REFCNT(result_live); + assert(ref_count == 1); + ref_count = Py_REFCNT(result); + assert(ref_count == 3); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 3); + assert(get_item == value); + + Py_DECREF(container); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value); + Py_DECREF(value_default); + Py_DECREF(result_live); + + assert(!PyErr_Occurred()); +} + +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +#pragma mark Dictionaries - getters + +void dbg_PyDict_GetItem(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + // No Key in the dictionary, no exception set. + assert(!PyErr_Occurred()); + get_item = PyDict_GetItem(container, key); + assert(get_item == NULL); + assert(!PyErr_Occurred()); + + // Set a value + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + Py_DECREF(container); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(key); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +void dbg_PyDict_GetItemRef(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + // Create something for result to point to and check it is abandoned. + PyObject *dummy_result = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(dummy_result); + assert(ref_count == 1); + PyObject *result = dummy_result; + + // No Key in the dictionary, no exception set. + assert(!PyErr_Occurred()); + int ret_val = PyDict_GetItemRef(container, key, &result); + assert(!PyErr_Occurred()); + assert(ret_val == 0); + assert(result == NULL); + ref_count = Py_REFCNT(dummy_result); + assert(ref_count == 1); + + // Set a value + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + assert(!PyErr_Occurred()); + ret_val = PyDict_GetItemRef(container, key, &result); + assert(!PyErr_Occurred()); + assert(ret_val == 1); + // value reference count has been incremented. + assert(result == value); + ref_count = Py_REFCNT(result); + assert(ref_count == 3); + + Py_DECREF(container); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value); + Py_DECREF(dummy_result); + + assert(!PyErr_Occurred()); +} + +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +/** + * This tests PyDict_GetItemWithError which contrary to the Python documentation + * does *not* set an exception if the key exists. + */ +void dbg_PyDict_GetItemWithError_fails(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + // No Key in the dictionary, exception set. + assert(!PyErr_Occurred()); + get_item = PyDict_GetItemWithError(container, key); + assert(get_item == NULL); + /* This is correct, the key is absent. */ + assert(!PyErr_Occurred()); + + /* So what error conditinos are handled? + * Firstly this will segfault. */ +#if 0 + assert(!PyErr_Occurred()); + get_item = PyDict_GetItemWithError(container, NULL); + assert(get_item == NULL); + assert(PyErr_Occurred()); +#endif + + PyObject *new_container = PyList_New(0); + assert(!PyErr_Occurred()); + get_item = PyDict_GetItemWithError(new_container, key); + assert(get_item == NULL); + assert(PyErr_Occurred()); + PyErr_Print(); /* Clears exception. */ + Py_DECREF(new_container); + + Py_DECREF(container); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + Py_DECREF(key); + + assert(!PyErr_Occurred()); +} + +#pragma mark - Dictionaries - deleters + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +// PyDict_Pop +// int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) +// https://docs.python.org/3/c-api/dict.html#c.PyDict_Pop +void dbg_PyDict_Pop_key_present(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 2); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + PyObject *get_item = PyDict_GetItem(container, key); + assert(get_item == value); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + + assert(PyDict_GET_SIZE(container) == 1); + + PyObject *result = NULL; + int return_value = PyDict_Pop(container, key, &result); + if (return_value != 1) { + assert(0); + } + + assert(PyDict_GET_SIZE(container) == 0); + + assert(result == value); + + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + ref_count = Py_REFCNT(result); + assert(ref_count == 2); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + assert(get_item == value); + + Py_DECREF(container); + + /* Dupe of above as the container is empty. */ + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + ref_count = Py_REFCNT(result); + assert(ref_count == 2); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 2); + assert(get_item == value); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_Pop_key_absent(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + assert(PyDict_GET_SIZE(container) == 0); + + /* Not inserted into the dict, just used so that result references it. */ + PyObject *dummy_value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(dummy_value); + assert(ref_count == 1); + + PyObject *result = dummy_value; + int return_value = PyDict_Pop(container, key, &result); + if (return_value != 0) { + assert(0); + } + + assert(PyDict_GET_SIZE(container) == 0); + + assert(result == NULL); + + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(dummy_value); + assert(ref_count == 1); + + Py_DECREF(container); + + /* Dupe of above as the container is empty. */ + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + ref_count = Py_REFCNT(dummy_value); + assert(ref_count == 1); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(dummy_value); + + assert(!PyErr_Occurred()); +} + +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +#pragma mark - Sets + +void dbg_PySet_Add(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PySet_New(NULL); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(PySet_GET_SIZE(container) == 0); + + if (PySet_Add(container, value)) { + assert(0); + } + assert(PySet_GET_SIZE(container) == 1); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + assert(PySet_Contains(container, value) == 1); + + // Now insert the same object again, dupe of the code above. + if (PySet_Add(container, value)) { + assert(0); + } + assert(PySet_GET_SIZE(container) == 1); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + assert(PySet_Contains(container, value) == 1); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* Clean up. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PySet_Discard(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PySet_New(NULL); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(PySet_GET_SIZE(container) == 0); + + if (PySet_Add(container, value)) { + assert(0); + } + assert(PySet_GET_SIZE(container) == 1); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + assert(PySet_Contains(container, value) == 1); + + if (PySet_Discard(container, value) != 1) { + assert(0); + } + assert(PySet_GET_SIZE(container) == 0); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(PySet_Contains(container, value) == 0); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* Clean up. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PySet_Pop(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PySet_New(NULL); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + assert(PySet_GET_SIZE(container) == 0); + + if (PySet_Add(container, value)) { + assert(0); + } + assert(PySet_GET_SIZE(container) == 1); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + assert(PySet_Contains(container, value) == 1); + + // Now pop() + PyObject *popped_value = PySet_Pop(container); + + assert(popped_value == value); + + assert(PySet_GET_SIZE(container) == 0); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + assert(PySet_Contains(container, value) == 0); + + Py_DECREF(container); + + ref_count = Py_REFCNT(value); + assert(ref_count == 2); + + /* Clean up. */ + Py_DECREF(value); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +#pragma mark - Struct Sequence + +static PyStructSequence_Field struct_sequence_simple_type_fields[] = { + {"family_name", "Family name."}, + {"given_name", "Given name."}, + {NULL, NULL} +}; + +static PyStructSequence_Desc struct_sequence_simple_type_desc = { + "module.struct_sequence_simple", + ".", + struct_sequence_simple_type_fields, + 2, +}; + +static PyTypeObject *static_struct_sequence_simple_type = NULL; + +void dbg_PyStructSequence_simple_ctor(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + if (static_struct_sequence_simple_type == NULL) { + static_struct_sequence_simple_type = PyStructSequence_NewType(&struct_sequence_simple_type_desc); + } + assert(static_struct_sequence_simple_type != NULL); + /* Hmm the refcount is 8. */ +// ref_count = Py_REFCNT(example_type); +// assert(ref_count == 1); + + PyObject *instance = PyStructSequence_New(static_struct_sequence_simple_type); + + ref_count = Py_REFCNT(instance); + assert(ref_count == 1); + + /* Get an unset item. */ + PyObject *get_item = NULL; + get_item = PyStructSequence_GetItem(instance, 0); + assert(get_item == NULL); + + /* Now set items. */ + PyObject *set_item = NULL; + set_item = new_unique_string(__FUNCTION__, "NAME"); + PyStructSequence_SetItem(instance, 0, set_item); + ref_count = Py_REFCNT(set_item); + assert(ref_count == 1); + set_item = new_unique_string(__FUNCTION__, "GENDER"); + PyStructSequence_SetItem(instance, 1, set_item); + ref_count = Py_REFCNT(set_item); + assert(ref_count == 1); + + /* Get items. */ + get_item = PyStructSequence_GetItem(instance, 0); + assert(get_item != NULL); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + get_item = PyStructSequence_GetItem(instance, 1); + assert(get_item != NULL); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + /* Clean up. */ + Py_DECREF(instance); + Py_DECREF(static_struct_sequence_simple_type); +} + +void dbg_PyStructSequence_setitem_abandons(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + if (static_struct_sequence_simple_type == NULL) { + static_struct_sequence_simple_type = PyStructSequence_NewType(&struct_sequence_simple_type_desc); + } + assert(static_struct_sequence_simple_type != NULL); + /* Hmm the ref count is 7. */ +// ref_count = Py_REFCNT(example_type); +// assert(ref_count == 1); + + PyObject *instance = PyStructSequence_New(static_struct_sequence_simple_type); + + ref_count = Py_REFCNT(instance); + assert(ref_count == 1); + + /* Now set items. */ + PyObject *set_item = NULL; + set_item = new_unique_string(__FUNCTION__, "NAME"); + PyStructSequence_SetItem(instance, 0, set_item); + ref_count = Py_REFCNT(set_item); + assert(ref_count == 1); + /* Set it again. */ + PyStructSequence_SetItem(instance, 0, set_item); + ref_count = Py_REFCNT(set_item); + assert(ref_count == 1); + + /* Clean up. */ + Py_DECREF(instance); + Py_DECREF(static_struct_sequence_simple_type); +} + +PyDoc_STRVAR( + struct_sequence_n_in_sequence_too_large_docstring, + "This uses struct_sequence_simple_type_fields but n_in_sequence is 3 rather than 2." +); + +/* + * This uses struct_sequence_simple_type_fields but n_in_sequence is 3 rather than 2. + */ +static PyStructSequence_Desc struct_sequence_n_in_sequence_too_large_type_desc = { + "module.struct_sequence_n_in_sequence_too_large", + struct_sequence_n_in_sequence_too_large_docstring, + struct_sequence_simple_type_fields, + 3, +}; + +void dbg_PyStructSequence_n_in_sequence_too_large(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); /* Clears error. */ + return; + } + assert(!PyErr_Occurred()); +// Py_ssize_t ref_count; + static PyTypeObject *example_type = NULL; + + if (example_type == NULL) { + example_type = PyStructSequence_NewType(&struct_sequence_n_in_sequence_too_large_type_desc); + } + assert(example_type == NULL); + assert(PyErr_Occurred()); + /* TypeError: tp_basicsize for type 'module.struct_sequence_n_in_sequence_too_large' (16) is too small for base 'tuple' (24). */ + fprintf(stderr, "%s(): On exit PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); /* Clears error. */ +} + + +void dbg_PyStructSequence_with_unnamed_field(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyStructSequence_Field struct_sequence_with_unnamed_fields[] = { + {"family_name", "Family name."}, + /* Use NULL then replace with PyStructSequence_UnnamedField + * otherwise get an error "initializer element is not a compile-time constant" */ + {"given_name", "Given name."}, + {PyStructSequence_UnnamedField, "Documentation for an unnamed field."}, + {NULL, NULL} + }; +// struct_sequence_with_unnamed_fields[2].name = PyStructSequence_UnnamedField; + + PyStructSequence_Desc struct_sequence_with_unnamed_field_type_desc = { + "module.struct_sequence_simple_with_unnamed_field", + "Documentation.", + struct_sequence_with_unnamed_fields, + 2, + }; + + PyTypeObject *example_type = NULL; + if (example_type == NULL) { + example_type = PyStructSequence_NewType(&struct_sequence_with_unnamed_field_type_desc); + } + assert(example_type != NULL); + /* Hmm. Refcount is 8. */ +// ref_count = Py_REFCNT(example_type); +// assert(ref_count == 1); + + PyObject *instance = PyStructSequence_New(example_type); + + ref_count = Py_REFCNT(instance); + assert(ref_count == 1); + + /* Get an unset item. */ + PyObject *get_item = NULL; + get_item = PyStructSequence_GetItem(instance, 0); + assert(get_item == NULL); + + /* Now set items. */ + PyObject *set_item = NULL; + set_item = new_unique_string(__FUNCTION__, "NAME"); + PyStructSequence_SetItem(instance, 0, set_item); + ref_count = Py_REFCNT(set_item); + assert(ref_count == 1); + set_item = new_unique_string(__FUNCTION__, "GENDER"); + PyStructSequence_SetItem(instance, 1, set_item); + ref_count = Py_REFCNT(set_item); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + fprintf(stdout, "Calling PyObject_Print(instance, stdout, 0);\n"); + PyObject_Print(instance, stdout, 0); + fprintf(stdout, "\n"); +// if (PyErr_Occurred()) { +// PyErr_Print(); +// } + assert(!PyErr_Occurred()); + fprintf(stdout, "Calling PyObject_Print(instance, stdout, Py_PRINT_RAW);\n"); + PyObject_Print(instance, stdout, Py_PRINT_RAW); + printf("\n"); +// if (PyErr_Occurred()) { +// PyErr_Print(); +// } + assert(!PyErr_Occurred()); + fprintf(stdout, "Calling PyObject_Print DONE\n"); + + /* Get items. */ + get_item = PyStructSequence_GetItem(instance, 0); + assert(get_item != NULL); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + get_item = PyStructSequence_GetItem(instance, 1); + assert(get_item != NULL); + ref_count = Py_REFCNT(get_item); + assert(ref_count == 1); + + assert(!PyErr_Occurred()); + + /* Clean up. */ + Py_DECREF(instance); + Py_DECREF(example_type); +} + +#pragma mark - Code that sefgfaults + +#if ACCEPT_SIGSEGV + +void dbg_PyTuple_SetItem_SIGSEGV_on_same_value(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + int result = PyTuple_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_value = PyTuple_GetItem(container, 0); + assert(get_value == value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* This causes value to be free'd. */ + result = PyTuple_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + assert(ref_count != 1); + + fprintf(stderr, "%s(): Undefined behaviour, possible SIGSEGV %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + /* This may cause a SIGSEGV. */ + Py_DECREF(container); + fprintf(stderr, "%s(): SIGSEGV did not happen %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); +} + +void dbg_PyList_SetItem_SIGSEGV_on_same_value(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + int result = PyList_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + PyObject *get_value = PyList_GetItem(container, 0); + assert(get_value == value); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + /* This causes value to be free'd. */ + result = PyList_SetItem(container, 0, value); + assert(result == 0); + ref_count = Py_REFCNT(value); + assert(ref_count != 1); + + fprintf(stderr, "%s(): Undefined behaviour, possible SIGSEGV %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + /* This may cause a SIGSEGV. */ + Py_DECREF(container); + fprintf(stderr, "%s(): SIGSEGV did not happen %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); +} + +void dbg_PyDict_SetItem_SIGSEGV_on_key_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = NULL; + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + fprintf(stderr, "%s(): PyDict_SetItem() with NULL key causes SIGSEGV %s#%d:\n", + __FUNCTION__, __FILE_NAME__, __LINE__); + int result = PyDict_SetItem(container, key, value); + fprintf(stderr, "%s(): SIGSEGV did not happen %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + if (result) { + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); /* Clears the error. */ + } else { + assert(0); + } + ref_count = Py_REFCNT(value); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(value); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_SetItem_SIGSEGV_on_value_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *value = NULL; + + fprintf(stderr, "%s(): PyDict_SetItem() with NULL value causes SIGSEGV %s#%d:\n", + __FUNCTION__, __FILE_NAME__, __LINE__); + int result = PyDict_SetItem(container, key, value); + fprintf(stderr, "%s(): SIGSEGV did not happen %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + if (result) { + assert(PyErr_Occurred()); + fprintf(stderr, "%s(): PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); /* Clears the error. */ + } else { + assert(0); + } + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + + Py_DECREF(container); + Py_DECREF(key); + + assert(!PyErr_Occurred()); +} + +void dbg_PyDict_GetItem_key_NULL(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + Py_ssize_t ref_count; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + + PyObject *key = NULL; + + // No Key in the dictionary, no exception set. + assert(!PyErr_Occurred()); + get_item = PyDict_GetItem(container, key); + assert(get_item == NULL); + assert(!PyErr_Occurred()); + + Py_DECREF(container); + + assert(!PyErr_Occurred()); +} + +#endif // ACCEPT_SIGSEGV diff --git a/src/cpy/Containers/DebugContainers.h b/src/cpy/Containers/DebugContainers.h new file mode 100644 index 0000000..b217bca --- /dev/null +++ b/src/cpy/Containers/DebugContainers.h @@ -0,0 +1,103 @@ +// +// Created by Paul Ross on 15/12/2024. +// + +#ifndef PYTHONEXTENSIONPATTERNS_DEBUGCONTAINERS_H +#define PYTHONEXTENSIONPATTERNS_DEBUGCONTAINERS_H + +#include "pyextpatt_util.h" + +#define ACCEPT_SIGSEGV 0 + +#pragma mark - Tuples +void dbg_PyTuple_SetItem_steals(void); +void dbg_PyTuple_SET_ITEM_steals(void); +void dbg_PyTuple_SetItem_steals_replace(void); +void dbg_PyTuple_SET_ITEM_steals_replace(void); +void dbg_PyTuple_SetItem_replace_with_same(void); +void dbg_PyTuple_SET_ITEM_replace_with_same(void); +void dbg_PyTuple_SetIem_NULL(void); +void dbg_PyTuple_SET_ITEM_NULL(void); +void dbg_PyTuple_SetIem_NULL_SetItem(void); +void dbg_PyTuple_SET_ITEM_NULL_SET_ITEM(void); +void dbg_PyTuple_SetItem_fails_not_a_tuple(void); +void dbg_PyTuple_SetItem_fails_out_of_range(void); +void dbg_PyTuple_PyTuple_Pack(void); +void dbg_PyTuple_Py_BuildValue(void); +#pragma mark - Lists +void dbg_PyList_SetItem_steals(void); +void dbg_PyList_SET_ITEM_steals(void); +void dbg_PyList_SetItem_steals_replace(void); +void dbg_PyList_SET_ITEM_steals_replace(void); +void dbg_PyList_SetItem_replace_with_same(void); +void dbg_PyList_SET_ITEM_replace_with_same(void); +void dbg_PyList_SetIem_NULL(void); +void dbg_PyList_SET_ITEM_NULL(void); +void dbg_PyList_SetIem_NULL_SetItem(void); +void dbg_PyList_SET_ITEM_NULL_SET_ITEM(void); +void dbg_PyList_SetItem_fails_not_a_tuple(void); +void dbg_PyList_SetItem_fails_out_of_range(void); +void dbg_PyList_Append(void); +void dbg_PyList_Append_fails_not_a_list(void); +void dbg_PyList_Append_fails_NULL(void); +void dbg_PyList_Insert(void); +void dbg_PyList_Insert_Is_Truncated(void); +void dbg_PyList_Insert_Negative_Index(void); +void dbg_PyList_Insert_fails_not_a_list(void); +void dbg_PyList_Insert_fails_NULL(void); +void dbg_PyList_Py_BuildValue(void); + +#pragma mark - Dictionaries - setters +void dbg_PyDict_SetItem_increments(void); + +#if ACCEPT_SIGSEGV +void dbg_PyDict_SetItem_NULL_key(void); +void dbg_PyDict_SetItem_NULL_value(void); +#endif // ACCEPT_SIGSEGV + +void dbg_PyDict_SetItem_fails_not_a_dict(void); +void dbg_PyDict_SetItem_fails_not_hashable(void); +void dbg_PyDict_SetDefault_default_unused(void); +void dbg_PyDict_SetDefault_default_used(void); +void dbg_PyDict_SetDefaultRef_default_unused(void); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 +void dbg_PyDict_SetDefaultRef_default_used(void); +void dbg_PyDict_SetDefaultRef_default_unused_result_non_null(void); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +#pragma mark - Dictionaries - getters +void dbg_PyDict_GetItem(void); +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 +void dbg_PyDict_GetItemRef(void); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 +void dbg_PyDict_GetItemWithError_fails(void); + +#pragma mark - Dictionaries - deleters + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 +void dbg_PyDict_Pop_key_present(void); +void dbg_PyDict_Pop_key_absent(void); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +#pragma mark - Sets + +void dbg_PySet_Add(void); +void dbg_PySet_Discard(void); +void dbg_PySet_Pop(void); + +#pragma mark - Struct Sequence + +void dbg_PyStructSequence_simple_ctor(void); +void dbg_PyStructSequence_setitem_abandons(void); +void dbg_PyStructSequence_n_in_sequence_too_large(void); +void dbg_PyStructSequence_with_unnamed_field(void); + +#if ACCEPT_SIGSEGV +void dbg_PyTuple_SetItem_SIGSEGV_on_same_value(void); +void dbg_PyList_SetItem_SIGSEGV_on_same_value(void); +void dbg_PyDict_SetItem_SIGSEGV_on_key_NULL(void); +void dbg_PyDict_SetItem_SIGSEGV_on_value_NULL(void); +void dbg_PyDict_GetItem_key_NULL(void); +#endif + +#endif //PYTHONEXTENSIONPATTERNS_DEBUGCONTAINERS_H diff --git a/src/cpy/CtxMgr/cCtxMgr.c b/src/cpy/CtxMgr/cCtxMgr.c new file mode 100644 index 0000000..fdf35c7 --- /dev/null +++ b/src/cpy/CtxMgr/cCtxMgr.c @@ -0,0 +1,142 @@ +/* A context manager example. */ + +/* MyObj objects */ + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +static const ssize_t BUFFER_LENGTH = (ssize_t)1024 * 1024 * 128; + +typedef struct { + PyObject_HEAD + /* Buffer created for the lifetime of the object. A memory check can show leaks. */ + char *buffer_lifetime; + /* Buffer created for the lifetime of the context. A memory check can show leaks. */ + char *buffer_context; +} ContextManager; + +/** Forward declaration. */ +static PyTypeObject ContextManager_Type; + +#define ContextManager_Check(v) (Py_TYPE(v) == &ContextManager_Type) + +static ContextManager * +ContextManager_new(PyObject *Py_UNUSED(arg)) { + ContextManager *self; + self = PyObject_New(ContextManager, &ContextManager_Type); + if (self == NULL) { + return NULL; + } + self->buffer_lifetime = malloc(BUFFER_LENGTH); + // Force an initialisation. + for (ssize_t i = 0; i < BUFFER_LENGTH; ++i) { + self->buffer_lifetime[i] = ' '; + } + self->buffer_context = NULL; +// fprintf(stdout, "%24s DONE REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); + return self; +} + +/* ContextManager methods */ +static void +ContextManager_dealloc(ContextManager *self) { +// fprintf(stdout, "%24s STRT REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); + free(self->buffer_lifetime); + self->buffer_lifetime = NULL; + assert(self->buffer_context == NULL); + PyObject_Del(self); +// fprintf(stdout, "%24s DONE REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); +} + +static PyObject * +ContextManager_enter(ContextManager *self, PyObject *Py_UNUSED(args)) { + assert(self->buffer_lifetime != NULL); + assert(self->buffer_context == NULL); +// fprintf(stdout, "%24s STRT REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); + self->buffer_context = malloc(BUFFER_LENGTH); + // Force an initialisation. + for (ssize_t i = 0; i < BUFFER_LENGTH; ++i) { + self->buffer_context[i] = ' '; + } + Py_INCREF(self); +// fprintf(stdout, "%24s DONE REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); + return (PyObject *)self; +} + +static PyObject * +ContextManager_exit(ContextManager *self, PyObject *Py_UNUSED(args)) { + assert(self->buffer_lifetime != NULL); + assert(self->buffer_context != NULL); +// fprintf(stdout, "%24s STRT REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); + free(self->buffer_context); + self->buffer_context = NULL; +// fprintf(stdout, "%24s DONE REFCNT = %zd\n", __FUNCTION__, Py_REFCNT(self)); + Py_RETURN_FALSE; +} + +static PyObject * +ContextManager_len_buffer_lifetime(ContextManager *self, PyObject *Py_UNUSED(args)) { + return Py_BuildValue("n", self->buffer_lifetime ? BUFFER_LENGTH : 0); +} + +static PyObject * +ContextManager_len_buffer_context(ContextManager *self, PyObject *Py_UNUSED(args)) { + return Py_BuildValue("n", self->buffer_context ? BUFFER_LENGTH : 0); +} + +static PyMethodDef ContextManager_methods[] = { + {"__enter__", (PyCFunction) ContextManager_enter, METH_NOARGS, + PyDoc_STR("__enter__() -> ContextManager")}, + {"__exit__", (PyCFunction) ContextManager_exit, METH_VARARGS, + PyDoc_STR("__exit__(exc_type, exc_value, exc_tb) -> bool")}, + {"len_buffer_lifetime", (PyCFunction) ContextManager_len_buffer_lifetime, METH_NOARGS, + PyDoc_STR("len_buffer_lifetime() -> int")}, + {"len_buffer_context", (PyCFunction) ContextManager_len_buffer_context, METH_NOARGS, + PyDoc_STR("len_buffer_context() -> int")}, + {NULL, NULL, 0, NULL} /* sentinel */ +}; + +static PyTypeObject ContextManager_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "cObject.ContextManager", + .tp_basicsize = sizeof(ContextManager), + .tp_dealloc = (destructor) ContextManager_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = ContextManager_methods, + .tp_new = (newfunc) ContextManager_new, +}; + +PyDoc_STRVAR(module_doc, "Example of a context manager."); + +static struct PyModuleDef cCtxMgr = { + PyModuleDef_HEAD_INIT, + .m_name = "cCtxMgr", + .m_doc = module_doc, + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_cCtxMgr(void) { + PyObject *m = NULL; + /* Create the module and add the functions */ + m = PyModule_Create(&cCtxMgr); + if (m == NULL) { + goto fail; + } + /* Finalize the type object including setting type of the new type + * object; doing it here is required for portability, too. */ + if (PyType_Ready(&ContextManager_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "ContextManager", (PyObject *) &ContextManager_Type)) { + goto fail; + } + if (PyModule_AddObject(m, "BUFFER_LENGTH", Py_BuildValue("n", BUFFER_LENGTH))) { + goto fail; + } + return m; +fail: + Py_XDECREF(m); + return NULL; +} diff --git a/src/cpy/Exceptions/cExceptions.c b/src/cpy/Exceptions/cExceptions.c new file mode 100644 index 0000000..2aca1c4 --- /dev/null +++ b/src/cpy/Exceptions/cExceptions.c @@ -0,0 +1,168 @@ +// +// cExceptions.c +// PythonExtensionPatterns +// +// Created by Paul Ross on 08/05/2014. +// Copyright (c) 2014-2025 Paul Ross. All rights reserved. +// + +#include "Python.h" + +/** Raise a simple exception. */ +static PyObject *raise_error(PyObject *Py_UNUSED(module)) { + PyErr_SetString(PyExc_ValueError, "Ooops."); + assert(PyErr_Occurred()); + return NULL; +} + +/** Raise an exception with a formatted message. */ +static PyObject *raise_error_formatted(PyObject *Py_UNUSED(module)) { + PyErr_Format(PyExc_ValueError, + "Can not read %d bytes when offset %d in byte length %d.", \ + 12, 25, 32 + ); + assert(PyErr_Occurred()); + return NULL; +} + +/** This illustrates the consequences of returning NULL but not setting an exception. */ +static PyObject *raise_error_bad(PyObject *Py_UNUSED(module)) { + PyErr_Clear(); + assert(!PyErr_Occurred()); + return NULL; +} + +/** Set an exception but fail to signal by returning non-NULL. */ +static PyObject *raise_error_silent(PyObject *Py_UNUSED(module)) { + PyErr_SetString(PyExc_ValueError, "ERROR: raise_error_silent()"); + assert(PyErr_Occurred()); + Py_RETURN_NONE; +} + +/** Test for an exception, possibly set by another function. */ +static PyObject *raise_error_silent_test(PyObject *Py_UNUSED(module)) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +/** Shows that second PyErr_SetString() is ignored. */ +static PyObject *raise_error_overwrite(PyObject *Py_UNUSED(module)) { + PyErr_SetString(PyExc_RuntimeError, "FORGOTTEN."); + PyErr_SetString(PyExc_ValueError, "ERROR: raise_error_overwrite()"); + assert(PyErr_Occurred()); + return NULL; +} + +/** Specialise exceptions base exception. */ +static PyObject *ExceptionBase = NULL; +/** Specialise exceptions derived from base exception. */ +static PyObject *SpecialisedError = NULL; + + +/** Raises a ExceptionBase. */ +static PyObject *raise_exception_base(PyObject *Py_UNUSED(module)) { + if (ExceptionBase) { + PyErr_Format(ExceptionBase, "One %d two %d three %d.", 1, 2, 3); + } else { + PyErr_SetString(PyExc_RuntimeError, "Can not raise exception, module not initialised correctly"); + } + return NULL; +} + +/** Raises a SpecialisedError. */ +static PyObject *raise_specialised_error(PyObject *Py_UNUSED(module)) { + if (SpecialisedError) { + PyErr_Format(SpecialisedError, "One %d two %d three %d.", 1, 2, 3); + } else { + PyErr_SetString(PyExc_RuntimeError, "Can not raise exception, module not initialised correctly"); + } + return NULL; +} + +static PyMethodDef cExceptions_methods[] = { + {"raise_error", (PyCFunction) raise_error, METH_NOARGS, + "Raise a simple exception." + }, + {"raise_error_fmt", (PyCFunction) raise_error_formatted, METH_NOARGS, + "Raise a formatted exception." + }, + {"raise_error_bad", (PyCFunction) raise_error_bad, METH_NOARGS, + "Signal an exception by returning NULL but fail to set an exception." + }, + {"raise_error_silent", (PyCFunction) raise_error_silent, METH_NOARGS, + "Set an exception but fail to signal it but returning non-NULL." + }, + {"raise_error_silent_test", (PyCFunction) raise_error_silent_test, METH_NOARGS, + "Raise if an exception is set otherwise returns None." + }, + { + "raise_error_overwrite", (PyCFunction) raise_error_overwrite, METH_NOARGS, + "Example of overwriting exceptions, a RuntimeError is set, then a ValueError. Only the latter is seen." + }, + { + "raise_exception_base", (PyCFunction) raise_exception_base, METH_NOARGS, "Raises a ExceptionBase." + }, + { + "raise_specialised_error", (PyCFunction) raise_specialised_error, METH_NOARGS, "Raises a SpecialisedError." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cExceptions_module = { + PyModuleDef_HEAD_INIT, + "cExceptions", + "Examples of raising exceptions.", + -1, + cExceptions_methods, + NULL, /* inquiry m_reload */ + NULL, /* traverseproc m_traverse */ + NULL, /* inquiry m_clear */ + NULL, /* freefunc m_free */ +}; + +PyMODINIT_FUNC +PyInit_cExceptions(void) { + PyObject *m = PyModule_Create(&cExceptions_module); + if (m == NULL) { + return NULL; + } + /* Initialise exceptions here. + * + * Firstly a base class exception that inherits from the builtin Exception. + * This is achieved by passing NULL as the PyObject* as the third argument. + * + * PyErr_NewExceptionWithDoc returns a new reference. + */ + ExceptionBase = PyErr_NewExceptionWithDoc( + "cExceptions.ExceptionBase", /* char *name */ + "Base exception class for the noddy module.", /* char *doc */ + NULL, /* PyObject *base, resolves to PyExc_Exception. */ + NULL /* PyObject *dict */); + /* Error checking: this is oversimplified as it should decref + * anything created above such as m. + */ + if (!ExceptionBase) { + return NULL; + } else { + PyModule_AddObject(m, "ExceptionBase", ExceptionBase); + } + /* Now a subclass exception that inherits from the base exception above. + * This is achieved by passing non-NULL as the PyObject* as the third argument. + * + * PyErr_NewExceptionWithDoc returns a new reference. + */ + SpecialisedError = PyErr_NewExceptionWithDoc( + "cExceptions.SpecialsiedError", /* char *name */ + "Some specialised problem description here.", /* char *doc */ + ExceptionBase, /* PyObject *base, declared above. */ + NULL /* PyObject *dict */); + if (!SpecialisedError) { + return NULL; + } else { + PyModule_AddObject(m, "SpecialisedError", SpecialisedError); + } + /* END: Initialise exceptions here. */ + return m; +} diff --git a/src/cpy/File/PythonFileWrapper.cpp b/src/cpy/File/PythonFileWrapper.cpp new file mode 100644 index 0000000..100b908 --- /dev/null +++ b/src/cpy/File/PythonFileWrapper.cpp @@ -0,0 +1,286 @@ +// +// Created by Paul Ross on 08/07/2021. +// + +#include "PythonFileWrapper.h" + +#include + +/** + * Macro that gets the given method and checks that it is callable. + * If not an ExceptionPythonFileObjectWrapper is thrown. + */ +#define EXTRACT_METHOD_AND_CHECK(name) \ + m_python_##name##_method = PyObject_GetAttrString(python_file_object, #name); /* New ref. */\ + if (!m_python_##name##_method) { \ + std::ostringstream oss; \ + oss << "PythonFileObjectWrapper: can not get method: " << #name << std::endl; \ + Py_XDECREF(python_file_object); \ + Py_XDECREF(m_python_read_method); \ + Py_XDECREF(m_python_write_method); \ + Py_XDECREF(m_python_seek_method); \ + Py_XDECREF(m_python_tell_method); \ + throw ExceptionPythonFileObjectWrapper(oss.str()); \ + } \ + if (!PyCallable_Check(m_python_##name##_method)) { \ + std::ostringstream oss; \ + oss << "PythonFileObjectWrapper: method: " << #name << " is not callable" << std::endl; \ + Py_XDECREF(m_python_file_object); \ + Py_XDECREF(m_python_read_method); \ + Py_XDECREF(m_python_write_method); \ + Py_XDECREF(m_python_seek_method); \ + Py_XDECREF(m_python_tell_method); \ + throw ExceptionPythonFileObjectWrapper(oss.str()); \ + } + +PythonFileObjectWrapper::PythonFileObjectWrapper(PyObject *python_file_object) : m_python_file_object( + python_file_object), + m_python_read_method(NULL), + m_python_write_method(NULL), + m_python_seek_method(NULL), + m_python_tell_method(NULL) { + assert(python_file_object); + Py_INCREF(m_python_file_object); + /* Get the read and write methods of the passed object */ + EXTRACT_METHOD_AND_CHECK(read); + EXTRACT_METHOD_AND_CHECK(write); + EXTRACT_METHOD_AND_CHECK(seek); + EXTRACT_METHOD_AND_CHECK(tell); +} + +int PythonFileObjectWrapper::read_py_write_cpp(Py_ssize_t number_of_bytes, std::iostream &ios) { + assert(!PyErr_Occurred()); + assert(m_python_file_object); + assert(m_python_read_method); + assert(m_python_write_method); + int ret = 0; +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d number_of_bytes=%ld\n", __FUNCTION__, __FILE__, __LINE__, number_of_bytes); +#endif + PyObject * read_args = Py_BuildValue("(i)", number_of_bytes); + PyObject * read_value = PyObject_Call(m_python_read_method, read_args, NULL); + if (read_value == NULL) { + ret = -1; + goto except; + } else { + /* Check for EOF */ + if (number_of_bytes >= 0 && PySequence_Length(read_value) != number_of_bytes) { + ret = -2; /* Signal EOF. */ + goto except; + } + if (PyBytes_Check(read_value)) { + ios.write(PyBytes_AsString(read_value), PyBytes_Size(read_value)); + } else if (PyUnicode_Check(read_value)) { + Py_ssize_t size; + const char *buffer = PyUnicode_AsUTF8AndSize(read_value, &size); + ios.write(buffer, size); + } else { + ret = -3; + goto except; + } + } + goto finally; + except: + /* Handle every abnormal condition and clean up. */ + assert(ret); + finally: + /* Clean up under normal conditions and return an appropriate value. */ + Py_XDECREF(read_args); + Py_XDECREF(read_value); +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d ret=%d\n", __FUNCTION__, __FILE__, __LINE__, ret); +#endif + return ret; +} + +int PythonFileObjectWrapper::read_cpp_write_py(std::iostream &ios, Py_ssize_t number_of_bytes) { + assert(!PyErr_Occurred()); + assert(m_python_file_object); + assert(m_python_read_method); + assert(m_python_write_method); + int ret = 0; + PyObject *py_bytes = NULL; + PyObject *write_args = NULL; + PyObject *write_result = NULL; +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d number_of_bytes=%ld\n", __FUNCTION__, __FILE__, __LINE__, number_of_bytes); +#endif + if (!ios.good()) { + PyErr_SetString(PyExc_ValueError, "C++ stream not capable of being read."); + goto except; + } + // Read from ios, write to Python file. + // Create a Python bytes object, read into it. + py_bytes = PyBytes_FromStringAndSize(NULL, number_of_bytes); + ios.read(PyBytes_AsString(py_bytes), number_of_bytes); + if (!ios.good()) { + PyErr_SetString(PyExc_ValueError, "Can not read from C++ stream."); + goto except; + } + write_args = Py_BuildValue("(O)", py_bytes); + write_result = PyObject_Call(m_python_write_method, write_args, NULL); + if (write_result == NULL) { + ret = -1; + goto except; + } + if (PyLong_AsLong(write_result) != number_of_bytes) { + ret = -2; + goto except; + } + goto finally; + except: + /* Handle every abnormal condition and clean up. */ + assert(ret); + finally: + /* Clean up under normal conditions and return an appropriate value. */ + Py_XDECREF(py_bytes); + Py_XDECREF(write_args); + Py_XDECREF(write_result); +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d ret=%d\n", __FUNCTION__, __FILE__, __LINE__, ret); +#endif + return ret; +} + + +int PythonFileObjectWrapper::read(Py_ssize_t number_of_bytes, std::vector &result) { + assert(!PyErr_Occurred()); + assert(m_python_file_object); + assert(m_python_read_method); + assert(m_python_write_method); + int ret = 0; +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d number_of_bytes=%ld\n", __FUNCTION__, __FILE__, __LINE__, number_of_bytes); +#endif + result.clear(); + PyObject * read_args = Py_BuildValue("(i)", number_of_bytes); + PyObject * read_value = PyObject_Call(m_python_read_method, read_args, NULL); + if (read_value == NULL) { + ret = -1; + goto except; + } else { + /* Check for EOF */ + if (number_of_bytes >= 0 && PySequence_Length(read_value) != number_of_bytes) { + ret = -2; /* Signal EOF. */ + goto except; + } + const char *buffer; + Py_ssize_t size; + if (PyBytes_Check(read_value)) { + buffer = PyBytes_AsString(read_value); + size =PyBytes_Size(read_value); + } else if (PyUnicode_Check(read_value)) { + buffer = PyUnicode_AsUTF8AndSize(read_value, &size); + } else { + ret = -3; + goto except; + } + for (Py_ssize_t i = 0; i < size; ++i) { + result.push_back(buffer[i]); + } + } + goto finally; + except: + /* Handle every abnormal condition and clean up. */ + assert(ret); + finally: + /* Clean up under normal conditions and return an appropriate value. */ + Py_XDECREF(read_args); + Py_XDECREF(read_value); +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d ret=%d\n", __FUNCTION__, __FILE__, __LINE__, ret); +#endif + return ret; +} + +int PythonFileObjectWrapper::write(const char *buffer, Py_ssize_t number_of_bytes) { + assert(!PyErr_Occurred()); + assert(m_python_file_object); + assert(m_python_read_method); + assert(m_python_write_method); + int ret = 0; + PyObject * py_bytes = NULL; + PyObject * write_args = NULL; + PyObject * write_result = NULL; +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d number_of_bytes=%ld\n", __FUNCTION__, __FILE__, __LINE__, number_of_bytes); +#endif + // Create a Python bytes object, read into it. + py_bytes = PyBytes_FromStringAndSize(buffer, number_of_bytes); + write_args = Py_BuildValue("(O)", py_bytes); + write_result = PyObject_Call(m_python_write_method, write_args, NULL); + if (write_result == NULL) { + ret = -1; + goto except; + } + if (PyLong_AsLong(write_result) != number_of_bytes) { + ret = -2; + goto except; + } + goto finally; +except: + /* Handle every abnormal condition and clean up. */ + assert(ret); +finally: + /* Clean up under normal conditions and return an appropriate value. */ + Py_XDECREF(py_bytes); + Py_XDECREF(write_args); + Py_XDECREF(write_result); +#if DEBUG_PYEXT_COMMON + fprintf(stdout, "%s(): %s#%d ret=%d\n", __FUNCTION__, __FILE__, __LINE__, ret); +#endif + return ret; +} + +long PythonFileObjectWrapper::seek(Py_ssize_t pos, int whence) { + assert(!PyErr_Occurred()); + assert(m_python_file_object); + assert(m_python_seek_method); + + PyObject * arguments = Py_BuildValue("ni", pos, whence); + PyObject * result = PyObject_Call(m_python_seek_method, arguments, NULL); + return PyLong_AsLong(result); +} + +long PythonFileObjectWrapper::tell() { + assert(!PyErr_Occurred()); + assert(m_python_file_object); + assert(m_python_tell_method); + + PyObject * result = PyObject_CallNoArgs(m_python_tell_method); + return PyLong_AsLong(result); +} + +std::string PythonFileObjectWrapper::str_pointers() const { + std::ostringstream oss; + oss << "PythonFileObjectWrapper:" << std::endl; + oss << "m_python_file_object " << std::hex << m_python_file_object << " type: " + << Py_TYPE(m_python_file_object)->tp_name << " ref count=" << std::dec << m_python_file_object->ob_refcnt + << std::endl; + oss << "m_python_read_method " << std::hex << m_python_read_method << " type: " + << Py_TYPE(m_python_read_method)->tp_name << " ref count=" << std::dec << m_python_read_method->ob_refcnt + << std::endl; + oss << "m_python_write_method " << std::hex << m_python_write_method << " type: " + << Py_TYPE(m_python_write_method)->tp_name << " ref count=" << std::dec << m_python_write_method->ob_refcnt + << std::endl; + oss << "m_python_seek_method " << std::hex << m_python_seek_method << " type: " + << Py_TYPE(m_python_seek_method)->tp_name << " ref count=" << std::dec << m_python_seek_method->ob_refcnt + << std::endl; + oss << "m_python_tell_method " << std::hex << m_python_tell_method << " type: " + << Py_TYPE(m_python_tell_method)->tp_name << " ref count=" << std::dec << m_python_tell_method->ob_refcnt + << std::endl; + return {oss.str()}; +} + +PyObject *PythonFileObjectWrapper::py_str_pointers() const { + std::string str_result = str_pointers(); + return PyBytes_FromStringAndSize(str_result.c_str(), (Py_ssize_t) str_result.size()); +} + +PythonFileObjectWrapper::~PythonFileObjectWrapper() { + Py_XDECREF(m_python_read_method); + Py_XDECREF(m_python_write_method); + Py_XDECREF(m_python_seek_method); + Py_XDECREF(m_python_tell_method); + Py_XDECREF(m_python_file_object); +} diff --git a/src/cpy/File/PythonFileWrapper.h b/src/cpy/File/PythonFileWrapper.h new file mode 100644 index 0000000..c33008f --- /dev/null +++ b/src/cpy/File/PythonFileWrapper.h @@ -0,0 +1,80 @@ +// +// Created by Paul Ross on 08/07/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHONFILEWRAPPER_H +#define PYTHONEXTENSIONSBASIC_PYTHONFILEWRAPPER_H +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +#include +#include +#include +#include +//#include + +class ExceptionPythonFileObjectWrapper : public std::exception { +public: + explicit ExceptionPythonFileObjectWrapper(std::string in_msg) : m_msg(std::move(in_msg)) {} + + [[nodiscard]] const std::string &message() const { return m_msg; } + + [[nodiscard]] const char *what() const + + noexcept override{return m_msg.c_str();} +protected: + std::string m_msg{}; +}; + + +/// Class that is created with a PyObject* that looks like a Python File. +/// This can then read from that file object ans write to a user provided C++ stream or read from a user provided C++ +/// stream and write to the give Python file like object. +class PythonFileObjectWrapper { +public: + explicit PythonFileObjectWrapper(PyObject *python_file_object); + + /// Read from a Python file and write to the C++ stream. + /// Return zero on success, non-zero on failure. + int read_py_write_cpp(Py_ssize_t number_of_bytes, std::iostream &ios); + + /// Read from a C++ stream and write to a Python file. + /// Return zero on success, non-zero on failure. + int read_cpp_write_py(std::iostream &ios, Py_ssize_t number_of_bytes); + + /// Read a number of bytes from a Python file and load them into the result. + /// Return zero on success, non-zero on failure. + int read(Py_ssize_t number_of_bytes, std::vector &result); + + /// Write a number of bytes to a Python file. + /// Return zero on success, non-zero on failure. + int write(const char *buffer, Py_ssize_t number_of_bytes); + + /// Move the file pointer to the given position. + /// whence is: + /// 0 – start of the stream (the default); offset should be zero or positive. + /// 1 – current stream position; offset may be negative. + /// 2 – end of the stream; offset is usually negative. + /// Returns the new absolute position. + long seek(Py_ssize_t pos, int whence = 0); + + /// Returns the current absolute position. + long tell(); + /// Returns a multi-line string that describes the class state. + std::string str_pointers() const; + /// Returns a Python multi-line bytes object that describes the class state. + PyObject *py_str_pointers() const; + /// Destructor, this decrements the held references. + virtual ~PythonFileObjectWrapper(); + +protected: + PyObject *m_python_file_object = NULL; + PyObject *m_python_read_method = NULL; + PyObject *m_python_write_method = NULL; + PyObject *m_python_seek_method = NULL; + PyObject *m_python_tell_method = NULL; +}; + +#endif //PYTHONEXTENSIONSBASIC_PYTHONFILEWRAPPER_H diff --git a/src/cpy/File/cFile.cpp b/src/cpy/File/cFile.cpp new file mode 100644 index 0000000..ebc95da --- /dev/null +++ b/src/cpy/File/cFile.cpp @@ -0,0 +1,303 @@ +// +// cFile.c +// PythonExtensionPatterns +// +// Created by Paul Ross on 10/07/2024. +// Copyright (c) 2024 Paul Ross. All rights reserved. +// + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" +#include "PythonFileWrapper.h" +#include "time.h" + +#define FPRINTF_DEBUG 0 + +/** Example of changing a Python string representing a file path to a C string and back again. + * + * The Python signature is: + * + * def parse_filesystem_argument(path: typing.Union[str, pathlib.Path]) -> str: + */ +static PyObject * +parse_filesystem_argument(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + assert(!PyErr_Occurred()); + assert(args || kwargs); + + PyBytesObject *py_path = NULL; + char *c_path = NULL; + Py_ssize_t path_size; + PyObject *ret = NULL; + + /* Parse arguments */ + static const char *kwlist[] = {"path", NULL}; + /* Can be optional output path with "|O&". */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&", const_cast(kwlist), PyUnicode_FSConverter, + &py_path)) { + goto except; + } + /* Check arguments. */ + assert(py_path); + /* Grab a reference to the internal bytes buffer. */ + if (PyBytes_AsStringAndSize((PyObject *) py_path, &c_path, &path_size)) { + /* Should have a TypeError or ValueError. */ + assert(PyErr_Occurred()); + assert(PyErr_ExceptionMatches(PyExc_TypeError) + || PyErr_ExceptionMatches(PyExc_ValueError)); + goto except; + } + assert(c_path); + /* Use the C path. */ + + /* Now convert the C path to a Python object, a string. */ + ret = PyUnicode_DecodeFSDefaultAndSize(c_path, path_size); + if (!ret) { + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + // Assert all temporary locals are NULL and thus have been transferred if used. + Py_XDECREF(py_path); + return ret; +} + + +/** + * Take a Python file object and and an integer and read that number of bytes and access this data in C. + * This returns the bytes read as a bytes object. + * + * Python signature: + * + * def read_python_file_to_c(file_object: typing.IO, size: int = -1) -> bytes: + */ +static PyObject * +read_python_file_to_c(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static const char *kwlist[] = {"file_object", "size", NULL}; + PyObject *py_file_object = NULL; + Py_ssize_t bytes_to_read = -1; + PyObject *py_read_meth = NULL; + PyObject *py_read_args = NULL; + PyObject *py_read_data = NULL; + char *c_bytes_data = NULL; + PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|n", (char **) (kwlist), + &py_file_object, &bytes_to_read)) { + return NULL; + } +#if FPRINTF_DEBUG + fprintf(stdout, "Got a file object of type \"%s\" and bytes to read of %ld\n", Py_TYPE(py_file_object)->tp_name, + bytes_to_read); +#endif + // Check that this is a readable file, well does it have a read method? + /* Get the read method of the passed object */ + py_read_meth = PyObject_GetAttrString(py_file_object, "read"); // New reference + if (py_read_meth == NULL) { + PyErr_Format(PyExc_ValueError, + "Argument of type %s does not have a read() method.", + Py_TYPE(py_file_object)->tp_name); + goto except; + } +#if FPRINTF_DEBUG + fprintf(stdout, "Have read attribute of type \"%s\"\n", Py_TYPE(py_read_meth)->tp_name); +#endif + if (!PyCallable_Check(py_read_meth)) { + PyErr_Format(PyExc_ValueError, + "read attribute of type %s is not callable.", + Py_TYPE(py_file_object)->tp_name); + goto except; + } +#if FPRINTF_DEBUG + fprintf(stdout, "Read attribute is callable.\n"); +#endif + // Call read(VisibleRecord::NUMBER_OF_HEADER_BYTES) to get a Python bytes object. + py_read_args = Py_BuildValue("(i)", bytes_to_read); + if (!py_read_args) { + goto except; + } + // This should advance that readable file pointer. + py_read_data = PyObject_Call(py_read_meth, py_read_args, NULL); + if (py_read_data == NULL) { + goto except; + } +#if FPRINTF_DEBUG + fprintf(stdout, "read_data is type \"%s\"\n", Py_TYPE(py_read_data)->tp_name); +#endif + /* Check for EOF */ + if (bytes_to_read >= 0 && PySequence_Length(py_read_data) != bytes_to_read) { + assert(PyErr_Occurred()); + PyErr_Format(PyExc_IOError, + "Reading file object gives EOF. Requested bytes %ld, got %ld.", + bytes_to_read, PySequence_Length(py_read_data)); + goto except; + } +#if FPRINTF_DEBUG + fprintf(stdout, "read_data is length is: %ld\n", PySequence_Length(py_read_data)); +#endif + c_bytes_data = PyBytes_AsString(py_read_data); + if (c_bytes_data == NULL) { + // TypeError already set. + goto except; + } +#if FPRINTF_DEBUG + fprintf(stdout, "Data is \"%s\"\n", c_bytes_data); +#endif + ret = py_read_data; + goto finally; +except: + /* Handle every abnormal condition and clean up. */ + assert(PyErr_Occurred()); + ret = NULL; +finally: + /* Clean up under normal conditions and return an appropriate value. */ + Py_XDECREF(py_read_meth); + Py_XDECREF(py_read_args); + return ret; +} + + +/** + * Take a Python bytes object, extract the bytes as a C char* and write to the python file object. + * This returns the number of bytes written. + * + * Python signature: + * + * def write_bytes_to_python_file(bytes_to_write: bytes, file_object: typing.IO) -> int: + */ +static PyObject * +write_bytes_to_python_file(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static const char *kwlist[] = {"bytes_to_write", "file_object", NULL}; + PyObject *py_file_object = NULL; + Py_buffer c_buffer; + PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "y*O", (char **) (kwlist), + &c_buffer, &py_file_object)) { + return NULL; + } +#if FPRINTF_DEBUG + fprintf(stdout, "Calling PyFile_WriteString() with bytes \"%s\"\n", (char *)c_buffer.buf); +#endif + /* NOTE: PyFile_WriteString() creates a unicode string and then calls PyFile_WriteObject() + * so the py_file_object must be capable of writing strings. */ + int result = PyFile_WriteString((char *)c_buffer.buf, py_file_object); + if (result != 0) { + goto except; + } + ret = Py_BuildValue("n", c_buffer.len); + goto finally; +except: + assert(PyErr_Occurred()); + ret = NULL; +finally: + return ret; +} + +/** + * Wraps a Python file object. + */ +static PyObject * +wrap_python_file(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static const char *kwlist[] = {"file_object", NULL}; + PyObject *py_file_object = NULL; +// PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", (char **) (kwlist), + &py_file_object)) { + return NULL; + } + PythonFileObjectWrapper py_file_wrapper(py_file_object); + + /* Exercise ths wrapper by writing, reading etc. */ + py_file_wrapper.write("Test write to python file", 25); + return py_file_wrapper.py_str_pointers(); +} + +#if 0 +/** + * Returns an integer file descriptor from a Python file object. + */ +int python_file_object_as_file_description(PyObject *op) { + int fd = PyObject_AsFileDescriptor(op); + if (fd < 0) { + return -1; + } + return fd; +} + +/** fd is an already open file. */ +PyObject *c_file_descriptor_as_python_file(int fd, const char *filename) { + PyObject *op = PyFile_FromFd(fd, filename, "r", -1, NULL, NULL, NULL, 1); + return op; +} + +/* + * fileno() man page: + * https://www.man7.org/linux/man-pages/man3/fileno.3.html + */ +PyObject *c_file_path_as_python_file(const char *filename, const char *mode) { + FILE *file = fopen(filename, mode); + int fd = fileno(file); + PyObject *op = PyFile_FromFd(fd, filename, "r", -1, NULL, NULL, NULL, 1); + return op; +} +#endif + +static PyMethodDef cFile_methods[] = { + { + "parse_filesystem_argument", + (PyCFunction) parse_filesystem_argument, + METH_VARARGS | METH_KEYWORDS, + "Parsing an argument that is a file path." + }, + { + "read_python_file_to_c", + (PyCFunction) read_python_file_to_c, + METH_VARARGS | METH_KEYWORDS, + "Read n bytes from a Python file." + }, + { + "write_bytes_to_python_file", + (PyCFunction) write_bytes_to_python_file, + METH_VARARGS | METH_KEYWORDS, + "Wrote bytes to a Python file." + }, + { + "wrap_python_file", + (PyCFunction) wrap_python_file, + METH_VARARGS | METH_KEYWORDS, + "Wrap a Python file." + }, + { + NULL, + NULL, + 0, + NULL + } /* Sentinel */ +}; + +static PyModuleDef cFile_module = { + PyModuleDef_HEAD_INIT, + "cFile", + "Examples of handling file paths and files in a Python 'C' extension.", + -1, + cFile_methods, + NULL, /* inquiry m_reload */ + NULL, /* traverseproc m_traverse */ + NULL, /* inquiry m_clear */ + NULL, /* freefunc m_free */ +}; + +PyMODINIT_FUNC PyInit_cFile(void) { + return PyModule_Create(&cFile_module); +} +/****************** END: Parsing arguments. ****************/ diff --git a/src/cpy/Iterators/cIterator.c b/src/cpy/Iterators/cIterator.c new file mode 100644 index 0000000..6a64d15 --- /dev/null +++ b/src/cpy/Iterators/cIterator.c @@ -0,0 +1,363 @@ +// +// Created by Paul Ross on 08/07/2021. +// +// Example of a iterator. +// +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +typedef struct { + PyObject_HEAD + long *array_long; + ssize_t size; +} SequenceOfLong; + +typedef struct { + PyObject_HEAD + PyObject *sequence; + size_t index; +} SequenceOfLongIterator; + +static PyObject * +SequenceOfLongIterator_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { + SequenceOfLongIterator *self; + self = (SequenceOfLongIterator *) type->tp_alloc(type, 0); + if (self != NULL) { + assert(!PyErr_Occurred()); + } + return (PyObject *) self; +} + +// Forward reference +static int is_sequence_of_long_type(PyObject *op); + +static int +SequenceOfLongIterator_init(SequenceOfLongIterator *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return -1; + } + if (!is_sequence_of_long_type(sequence)) { + PyErr_Format( + PyExc_ValueError, + "Argument must be a SequenceOfLongType, not type %s", + Py_TYPE(sequence)->tp_name + ); + return -2; + } + // Borrowed reference + // Keep the sequence alive as long as the iterator is alive. + // Decrement on iterator de-allocation. + Py_INCREF(sequence); + self->sequence = sequence; + self->index = 0; + return 0; +} + +static void +SequenceOfLongIterator_dealloc(SequenceOfLongIterator *self) { + // Decrement borrowed reference. + Py_XDECREF(self->sequence); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +SequenceOfLongIterator_next(SequenceOfLongIterator *self) { + size_t size = ((SequenceOfLong *) self->sequence)->size; + if (self->index < size) { + PyObject *ret = PyLong_FromLong(((SequenceOfLong *) self->sequence)->array_long[self->index]); + self->index += 1; + return ret; + } + // End iteration. + return NULL; +} + +static PyObject * +SequenceOfLongIterator___str__(SequenceOfLongIterator *self, PyObject *Py_UNUSED(ignored)) { + assert(!PyErr_Occurred()); + if (self->sequence) { + return PyUnicode_FromFormat( + "sequence, ((SequenceOfLong *) self->sequence)->size, self->index + ); + } else { + return PyUnicode_FromFormat( + "sequence, self->index + ); + } +} + +static PyTypeObject SequenceOfLongIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceOfLongIterator", + .tp_basicsize = sizeof(SequenceOfLongIterator), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceOfLongIterator_dealloc, + .tp_str = (reprfunc) SequenceOfLongIterator___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "SequenceOfLongIterator object.", + .tp_iter = PyObject_SelfIter, + .tp_iternext = (iternextfunc) SequenceOfLongIterator_next, + .tp_init = (initproc) SequenceOfLongIterator_init, + .tp_new = SequenceOfLongIterator_new, +}; + +static PyObject * +SequenceOfLong_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { + SequenceOfLong *self; + self = (SequenceOfLong *) type->tp_alloc(type, 0); + if (self != NULL) { + assert(!PyErr_Occurred()); + self->size = 0; + self->array_long = NULL; + } + return (PyObject *) self; +} + +static int +SequenceOfLong_init(SequenceOfLong *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return -1; + } + if (!PySequence_Check(sequence)) { + return -2; + } + self->size = PySequence_Length(sequence); + self->array_long = malloc(self->size * sizeof(long)); + if (!self->array_long) { + return -3; + } + for (Py_ssize_t i = 0; i < PySequence_Length(sequence); ++i) { + // New reference. + PyObject *py_value = PySequence_GetItem(sequence, i); + if (PyLong_Check(py_value)) { + self->array_long[i] = PyLong_AsLong(py_value); + Py_DECREF(py_value); + } else { + PyErr_Format( + PyExc_TypeError, + "Argument [%zd] must be a int, not type %s", + i, + Py_TYPE(sequence)->tp_name + ); + // Clean up on error. + free(self->array_long); + self->array_long = NULL; + Py_DECREF(py_value); + return -4; + } + } + return 0; +} + +static void +SequenceOfLong_dealloc(SequenceOfLong *self) { + free(self->array_long); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +SequenceOfLong_size(SequenceOfLong *self, PyObject *Py_UNUSED(ignored)) { + return Py_BuildValue("n", self->size); +} + +static PyObject * +SequenceOfLong_iter(SequenceOfLong *self) { + PyObject *ret = SequenceOfLongIterator_new(&SequenceOfLongIteratorType, NULL, NULL); + if (ret) { + PyObject *args = Py_BuildValue("(O)", self); + if (!args || SequenceOfLongIterator_init((SequenceOfLongIterator *) ret, args, NULL)) { + Py_DECREF(ret); + ret = NULL; + } + Py_DECREF(args); + } + return ret; +} + +static PyMethodDef SequenceOfLong_methods[] = { + { + "size", + (PyCFunction) SequenceOfLong_size, + METH_NOARGS, + "Return the size of the sequence." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +/* Sequence methods. */ +static Py_ssize_t +SequenceOfLong_len(PyObject *self) { + return ((SequenceOfLong *)self)->size; +} + +static PyObject * +SequenceOfLong_getitem(PyObject *self, Py_ssize_t index) { + Py_ssize_t my_index = index; + if (my_index < 0) { + my_index += SequenceOfLong_len(self); + } + if (my_index > SequenceOfLong_len(self)) { + PyErr_Format( + PyExc_IndexError, + "Index %ld is out of range for length %ld", + index, + SequenceOfLong_len(self) + ); + return NULL; + } + return PyLong_FromLong(((SequenceOfLong *)self)->array_long[my_index]); +} + +PySequenceMethods SequenceOfLong_sequence_methods = { + .sq_length = &SequenceOfLong_len, + .sq_item = &SequenceOfLong_getitem, +}; + +static PyObject * +SequenceOfLong___str__(SequenceOfLong *self, PyObject *Py_UNUSED(ignored)) { + assert(!PyErr_Occurred()); + return PyUnicode_FromFormat("", self->size); +} + +static PyTypeObject SequenceOfLongType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceOfLong", + .tp_basicsize = sizeof(SequenceOfLong), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceOfLong_dealloc, + .tp_as_sequence = &SequenceOfLong_sequence_methods, + .tp_str = (reprfunc) SequenceOfLong___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "Sequence of long integers.", + .tp_iter = (getiterfunc) SequenceOfLong_iter, +// .tp_iternext = (iternextfunc) SequenceOfLongIterator_next, + .tp_methods = SequenceOfLong_methods, + .tp_init = (initproc) SequenceOfLong_init, + .tp_new = SequenceOfLong_new, +}; + +static int +is_sequence_of_long_type(PyObject *op) { + return Py_TYPE(op) == &SequenceOfLongType; +} + +static PyObject * +iterate_and_print(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return NULL; + } +// if (!PyIter_Check(sequence)) { +// PyErr_Format(PyExc_TypeError, "Object of type %s does support the iterator protocol", +// Py_TYPE(sequence)->tp_name); +// return NULL; +// } + PyObject *iterator = PyObject_GetIter(sequence); + if (iterator == NULL) { + /* propagate error */ + assert(PyErr_Occurred()); + return NULL; + } + PyObject *item = NULL; + long index = 0; + fprintf(stdout, "%s:\n", __FUNCTION__ ); + while ((item = PyIter_Next(iterator))) { + /* do something with item */ + fprintf(stdout, "[%ld]: ", index); + if (PyObject_Print(item, stdout, Py_PRINT_RAW) == -1) { + /* Handle error. */ + Py_DECREF(item); + Py_DECREF(iterator); + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, + "Can not print an item of type %s", + Py_TYPE(sequence)->tp_name); + } + return NULL; + } + fprintf(stdout, "\n"); + ++index; + /* release reference when done */ + Py_DECREF(item); + } + Py_DECREF(iterator); + if (PyErr_Occurred()) { + /* propagate error */ + return NULL; + } + fprintf(stdout, "%s: DONE\n", __FUNCTION__ ); + fflush(stdout); + assert(!PyErr_Occurred()); + Py_RETURN_NONE; +} + +static PyMethodDef cIterator_methods[] = { + {"iterate_and_print", (PyCFunction) iterate_and_print, METH_VARARGS, + "Iteratee through the argument printing the values."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef iterator_cmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cIterator", + .m_doc = ( + "Example module that creates an extension type" + "that has forward and reverse iterators." + ), + .m_size = -1, + .m_methods = cIterator_methods, +}; + +PyMODINIT_FUNC +PyInit_cIterator(void) { + PyObject *m; + m = PyModule_Create(&iterator_cmodule); + if (m == NULL) { + return NULL; + } + + if (PyType_Ready(&SequenceOfLongType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&SequenceOfLongType); + if (PyModule_AddObject( + m, + "SequenceOfLong", + (PyObject *) &SequenceOfLongType) < 0 + ) { + Py_DECREF(&SequenceOfLongType); + Py_DECREF(m); + return NULL; + } + if (PyType_Ready(&SequenceOfLongIteratorType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&SequenceOfLongIteratorType); + // Not strictly necessary unless you need to expose this type. + // For type checking for example. + if (PyModule_AddObject( + m, + "SequenceOfLongIterator", + (PyObject *) &SequenceOfLongIteratorType) < 0 + ) { + Py_DECREF(&SequenceOfLongType); + Py_DECREF(&SequenceOfLongIteratorType); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/Logging/cLogging.c b/src/cpy/Logging/cLogging.c new file mode 100644 index 0000000..5020a63 --- /dev/null +++ b/src/cpy/Logging/cLogging.c @@ -0,0 +1,228 @@ +// Example of using Python's logging mechanism from C. +// Based on, and thanks to, an initial submission from https://github.com/nnathan +// See also https://docs.python.org/3/library/logging.html + +#define PPY_SSIZE_T_CLEAN + +#include +/* For va_start, va_end */ +#include + +/* logging levels defined by logging module + * From: https://docs.python.org/3/library/logging.html#logging-levels */ +#define LOGGING_DEBUG 10 +#define LOGGING_INFO 20 +#define LOGGING_WARNING 30 +#define LOGGING_ERROR 40 +#define LOGGING_CRITICAL 50 +#define LOGGING_EXCEPTION 60 + +/* This modules globals */ +static PyObject *g_logging_module = NULL; /* Initialise by PyInit_cLogging() below. */ +static PyObject *g_logger = NULL; + +/* Get a logger object from the logging module. */ +static PyObject *py_get_logger(char *logger_name) { + assert(g_logging_module); + PyObject *logger = NULL; + + logger = PyObject_CallMethod(g_logging_module, "getLogger", "s", logger_name); + if (logger == NULL) { + const char *err_msg = "failed to call logging.getLogger"; + PyErr_SetString(PyExc_RuntimeError, err_msg); + } + /* + fprintf(stdout, "%s()#%d logger=0x%p\n", __FUNCTION__, __LINE__, (void *)logger); + */ + return logger; +} + +/* main interface to logging function */ +static PyObject * +py_log_msg(int log_level, char *printf_fmt, ...) { + assert(g_logger); + assert(!PyErr_Occurred()); + PyObject *log_msg = NULL; + PyObject *ret = NULL; + va_list fmt_args; + /* + fprintf(stdout, "%s()#%d g_logger=0x%p\n", __FUNCTION__, __LINE__, (void *)g_logger); + fprintf(stdout, "%s()#%d log_level=%d print_fmt=\"%s\"\n", __FUNCTION__, __LINE__, log_level, printf_fmt); + */ + va_start(fmt_args, printf_fmt); + log_msg = PyUnicode_FromFormatV(printf_fmt, fmt_args); + va_end(fmt_args); + /* + fprintf(stdout, "%s()#%d log_message: \"", __FUNCTION__, __LINE__); + PyObject_Print(log_msg, stdout, Py_PRINT_RAW); + fprintf(stdout, "\"\n"); + */ + if (log_msg == NULL) { + /* fail. */ + ret = PyObject_CallMethod( + g_logger, + "critical", + "O", "Unable to create log message." + ); + } else { + /* call function depending on loglevel */ + switch (log_level) { + case LOGGING_DEBUG: + ret = PyObject_CallMethod(g_logger, "debug", "O", log_msg); + break; + case LOGGING_INFO: + ret = PyObject_CallMethod(g_logger, "info", "O", log_msg); + break; + case LOGGING_WARNING: + ret = PyObject_CallMethod(g_logger, "warning", "O", log_msg); + break; + case LOGGING_ERROR: + ret = PyObject_CallMethod(g_logger, "error", "O", log_msg); + break; + case LOGGING_CRITICAL: + ret = PyObject_CallMethod(g_logger, "critical", "O", log_msg); + break; + default: + ret = PyObject_CallMethod(g_logger, "critical", "O", log_msg); + break; + } + assert(!PyErr_Occurred()); + } + Py_DECREF(log_msg); + return ret; +} + +static PyObject * +py_log_message(PyObject *Py_UNUSED(module), PyObject *args) { + int log_level; + char *message; + + if (!PyArg_ParseTuple(args, "iz", &log_level, &message)) { + return NULL; + } + return py_log_msg(log_level, "%s", message); +} + +static PyObject * +py_log_set_level(PyObject *Py_UNUSED(module), PyObject *args) { + assert(g_logger); + PyObject *py_log_level; + + if (!PyArg_ParseTuple(args, "O", &py_log_level)) { + return NULL; + } + return PyObject_CallMethod(g_logger, "setLevel", "O", py_log_level); +} + +/** + * Returns a tuple of the file, line and function of the current Python frame. + * Returns (None, 0, None) on failure. + * @param _unused_module + * @return PyObject *, a tuple of three values. + */ +static PyObject * +py_file_line_function(PyObject *Py_UNUSED(module)) { + const unsigned char *file_name = NULL; + const char *func_name = NULL; + int line_number = 0; + + PyFrameObject *frame = PyEval_GetFrame(); + if (frame) { + file_name = PyUnicode_1BYTE_DATA(PyFrame_GetCode(frame)->co_filename); + line_number = PyFrame_GetLineNumber(frame); + func_name = (const char *) PyUnicode_1BYTE_DATA(PyFrame_GetCode(frame)->co_name); + } + /* Use 'z' that makes Python None if the string is NULL. */ + return Py_BuildValue("ziz", file_name, line_number, func_name); +} + +#define C_FILE_LINE_FUNCTION Py_BuildValue("sis", __FILE__, __LINE__, __FUNCTION__) + +static PyObject * +c_file_line_function(PyObject *Py_UNUSED(module)) { + return C_FILE_LINE_FUNCTION; +} + +static PyMethodDef logging_methods[] = { + { + "py_log_set_level", + (PyCFunction) py_log_set_level, + METH_VARARGS, + "Set the logging level." + }, + { + "log", + (PyCFunction) py_log_message, + METH_VARARGS, + "Log a message." + }, + { + "py_file_line_function", + (PyCFunction) py_file_line_function, + METH_NOARGS, + "Return the file, line and function name from the current Python frame." + }, + { + "c_file_line_function", + (PyCFunction) c_file_line_function, + METH_NOARGS, + "Return the file, line and function name from the current C code." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cLogging = { + PyModuleDef_HEAD_INIT, + .m_name = "cLogging", + .m_doc = "Logging module.", + .m_size = -1, + .m_methods = logging_methods, +}; + +PyMODINIT_FUNC PyInit_cLogging(void) { + PyObject *m = PyModule_Create(&cLogging); + if (!m) { + goto except; + } + g_logging_module = PyImport_ImportModule("logging"); + if (!g_logging_module) { + const char *err_msg = "failed to import 'logging'"; + PyErr_SetString(PyExc_ImportError, err_msg); + goto except; + } + g_logger = py_get_logger("cLogging"); + if (!g_logger) { + goto except; + } + /* Adding module globals */ + /* logging levels defined by logging module. */ + if (PyModule_AddIntConstant(m, "DEBUG", LOGGING_DEBUG)) { + goto except; + } + if (PyModule_AddIntConstant(m, "INFO", LOGGING_INFO)) { + goto except; + } + if (PyModule_AddIntConstant(m, "WARNING", LOGGING_WARNING)) { + goto except; + } + if (PyModule_AddIntConstant(m, "ERROR", LOGGING_ERROR)) { + goto except; + } + if (PyModule_AddIntConstant(m, "CRITICAL", LOGGING_CRITICAL)) { + goto except; + } + if (PyModule_AddIntConstant(m, "EXCEPTION", LOGGING_EXCEPTION)) { + goto except; + } + + goto finally; + except: + /* abnormal cleanup */ + /* cleanup logger references */ + Py_XDECREF(g_logging_module); + Py_XDECREF(g_logger); + Py_XDECREF(m); + m = NULL; + finally: + return m; +} diff --git a/src/cModuleGlobals.c b/src/cpy/ModuleGlobals/cModuleGlobals.c similarity index 61% rename from src/cModuleGlobals.c rename to src/cpy/ModuleGlobals/cModuleGlobals.c index 05f9d7f..b283bab 100644 --- a/src/cModuleGlobals.c +++ b/src/cpy/ModuleGlobals/cModuleGlobals.c @@ -3,7 +3,7 @@ // PythonExtensionPatterns // // Created by Paul Ross on 09/05/2014. -// Copyright (c) 2014 Paul Ross. All rights reserved. +// Copyright (c) 2014-2025 Paul Ross. All rights reserved. // /* This is the code used for the documentation at: @@ -15,204 +15,229 @@ #include "Python.h" +#define FPRINTF_DEBUG 0 + const char *NAME_INT = "INT"; const char *NAME_STR = "STR"; const char *NAME_LST = "LST"; const char *NAME_TUP = "TUP"; const char *NAME_MAP = "MAP"; -static PyObject *_print_global_INT(PyObject *pMod) { - PyObject *ret = NULL; +static PyObject *print_global_INT(PyObject *pMod) { + PyObject *ret = NULL; PyObject *pItem = NULL; - long val; - + /* Returns a new reference. */ pItem = PyObject_GetAttrString(pMod, NAME_INT); - if (! pItem) { + if (!pItem) { PyErr_Format(PyExc_AttributeError, "Module '%s' has no attibute '%s'.", \ PyModule_GetName(pMod), NAME_INT - ); + ); goto except; } +#if FPRINTF_DEBUG fprintf(stdout, "Integer: \"%s\" ", NAME_INT); PyObject_Print(pItem, stdout, 0); - val = PyLong_AsLong(pItem); + long val = PyLong_AsLong(pItem); fprintf(stdout, " C long: %ld ", val); fprintf(stdout, "\n"); +#endif - assert(! PyErr_Occurred()); + assert(!PyErr_Occurred()); Py_INCREF(Py_None); ret = Py_None; - goto finally; -except: + goto finally; + except: assert(PyErr_Occurred()); Py_XDECREF(pItem); - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; } -static PyObject *_print_global_INT_borrowed_ref(PyObject *pMod) { - PyObject *ret = NULL; +static PyObject *print_global_INT_borrowed_ref(PyObject *pMod) { + PyObject *ret = NULL; PyObject *pItem = NULL; - long val; assert(pMod); assert(PyModule_CheckExact(pMod)); - assert(! PyErr_Occurred()); + assert(!PyErr_Occurred()); +#if FPRINTF_DEBUG fprintf(stdout, "Module:\n"); PyObject_Print(pMod, stdout, 0); fprintf(stdout, "\n"); +#endif - /* NOTE: PyModule_GetDict(pMod); never fails and returns a borrowed - * reference. pItem is NULL or a borrowed reference. - */ - pItem = PyDict_GetItemString(PyModule_GetDict(pMod), NAME_INT); - if (! pItem) { + /* NOTE: PyModule_GetDict(pMod); never fails and returns a borrowed + * reference. pItem is NULL or a borrowed reference. + */ + pItem = PyDict_GetItemString(PyModule_GetDict(pMod), NAME_INT); + if (!pItem) { PyErr_Format(PyExc_AttributeError, "Module '%s' has no attibute '%s'.", \ PyModule_GetName(pMod), NAME_INT - ); + ); goto except; } Py_INCREF(pItem); + +#if FPRINTF_DEBUG fprintf(stdout, "Integer: \"%s\" ", NAME_INT); PyObject_Print(pItem, stdout, 0); - val = PyLong_AsLong(pItem); + long val = PyLong_AsLong(pItem); fprintf(stdout, " C long: %ld ", val); fprintf(stdout, "\n"); +#endif - assert(! PyErr_Occurred()); + assert(!PyErr_Occurred()); Py_INCREF(Py_None); ret = Py_None; - goto finally; -except: + goto finally; + except: assert(PyErr_Occurred()); - Py_XDECREF(ret); - ret = NULL; -finally: + Py_XDECREF(ret); + ret = NULL; + finally: Py_DECREF(pItem); - return ret; + return ret; } -static PyObject *_print_globals(PyObject *pMod) { - PyObject *ret = NULL; +static PyObject *print_globals(PyObject *pMod) { + PyObject *ret = NULL; PyObject *pItem = NULL; assert(pMod); assert(PyModule_CheckExact(pMod)); - assert(! PyErr_Occurred()); + assert(!PyErr_Occurred()); +#if FPRINTF_DEBUG fprintf(stdout, "cModuleGlobals:\n"); PyObject_Print(pMod, stdout, 0); fprintf(stdout, "\n"); +#endif /* Your code here...*/ - if (! _print_global_INT(pMod)) { - goto except; + if (!print_global_INT(pMod)) { + goto except; } - if (! _print_global_INT_borrowed_ref(pMod)) { - goto except; + if (!print_global_INT_borrowed_ref(pMod)) { + goto except; } pItem = PyObject_GetAttrString(pMod, NAME_STR); - if (! pItem) { + if (!pItem) { PyErr_Format(PyExc_AttributeError, "Module '%s' has no attibute '%s'.", \ PyModule_GetName(pMod), NAME_STR - ); + ); goto except; } +#if FPRINTF_DEBUG fprintf(stdout, " String: \"%s\" ", NAME_STR); PyObject_Print(pItem, stdout, 0); fprintf(stdout, "\n"); Py_DECREF(pItem); pItem = NULL; - +#endif + pItem = PyObject_GetAttrString(pMod, NAME_LST); - if (! pItem) { + if (!pItem) { PyErr_Format(PyExc_AttributeError, "Module '%s' has no attibute '%s'.", \ PyModule_GetName(pMod), NAME_LST - ); + ); goto except; } +#if FPRINTF_DEBUG fprintf(stdout, " List: \"%s\" ", NAME_LST); PyObject_Print(pItem, stdout, 0); fprintf(stdout, "\n"); +#endif + Py_DECREF(pItem); pItem = NULL; - pItem = PyObject_GetAttrString(pMod, NAME_MAP); - if (! pItem) { + if (!pItem) { PyErr_Format(PyExc_AttributeError, "Module '%s' has no attibute '%s'.", \ PyModule_GetName(pMod), NAME_MAP - ); + ); goto except; } +#if FPRINTF_DEBUG fprintf(stdout, " Map: \"%s\" ", NAME_MAP); PyObject_Print(pItem, stdout, 0); fprintf(stdout, "\n"); +#endif + Py_DECREF(pItem); pItem = NULL; - - assert(! PyErr_Occurred()); + + assert(!PyErr_Occurred()); Py_INCREF(Py_None); ret = Py_None; - goto finally; -except: + goto finally; + except: assert(PyErr_Occurred()); - Py_XDECREF(ret); - ret = NULL; -finally: - return ret; + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; } - static PyMethodDef cModuleGlobals_methods[] = { - {"print", (PyCFunction)_print_globals, METH_NOARGS, - "Access and print out th globals." - }, - {NULL, NULL, 0, NULL} /* Sentinel */ + {"print", (PyCFunction) print_globals, METH_NOARGS, + "Access and print out the globals." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ }; static PyModuleDef cModuleGlobals_module = { - PyModuleDef_HEAD_INIT, - "cModuleGlobals", - "Examples of global values in a module.", - -1, - cModuleGlobals_methods, /* cModuleGlobals_methods */ - NULL, /* inquiry m_reload */ - NULL, /* traverseproc m_traverse */ - NULL, /* inquiry m_clear */ - NULL, /* freefunc m_free */ + PyModuleDef_HEAD_INIT, + "cModuleGlobals", + "Examples of global values in a module.", + -1, + cModuleGlobals_methods, /* cModuleGlobals_methods */ + NULL, /* inquiry m_reload */ + NULL, /* traverseproc m_traverse */ + NULL, /* inquiry m_clear */ + NULL, /* freefunc m_free */ }; /* Add a dict of {str : int, ...}. * Returns 0 on success, 1 on failure. */ -int _add_map_to_module(PyObject *module) { +int add_map_to_module(PyObject *module) { int ret = 0; PyObject *pMap = NULL; - + PyObject *key = NULL; + PyObject *val = NULL; + pMap = PyDict_New(); - if (! pMap) { + if (pMap == NULL) { goto except; } /* Load map. */ - if (PyDict_SetItem(pMap, PyBytes_FromString("66"), PyLong_FromLong(66))) { + key = PyBytes_FromString("66"); + val = PyLong_FromLong(66); + if (PyDict_SetItem(pMap, key, val)) { goto except; } + Py_XDECREF(key); + Py_XDECREF(val); + key = PyBytes_FromString("123"); + val = PyLong_FromLong(123); if (PyDict_SetItem(pMap, PyBytes_FromString("123"), PyLong_FromLong(123))) { goto except; } + Py_XDECREF(key); + Py_XDECREF(val); /* Add map to module. */ if (PyModule_AddObject(module, NAME_MAP, pMap)) { goto except; @@ -220,40 +245,41 @@ int _add_map_to_module(PyObject *module) { ret = 0; goto finally; except: - Py_XDECREF(pMap); - ret = 1; + Py_XDECREF(pMap); + Py_XDECREF(key); + Py_XDECREF(val); + ret = 1; finally: - return ret; + return ret; } PyMODINIT_FUNC -PyInit_cModuleGlobals(void) -{ +PyInit_cModuleGlobals(void) { PyObject *m = NULL; - + m = PyModule_Create(&cModuleGlobals_module); - + if (m == NULL) { goto except; } - /* Adding module globals */ - if (PyModule_AddIntConstant(m, NAME_INT, 42)) { + /* Adding module globals */ + if (PyModule_AddIntConstant(m, NAME_INT, 42)) { goto except; - } + } if (PyModule_AddStringConstant(m, NAME_STR, "String value")) { goto except; } - if (PyModule_AddObject(m, NAME_TUP, Py_BuildValue("iii", 66, 68, 73))) { + if (PyModule_AddObject(m, NAME_TUP, Py_BuildValue("iii", 66, 68, 73))) { goto except; - } - if (PyModule_AddObject(m, NAME_LST, Py_BuildValue("[iii]", 66, 68, 73))) { + } + if (PyModule_AddObject(m, NAME_LST, Py_BuildValue("[iii]", 66, 68, 73))) { goto except; - } + } /* An invented convenience function for this dict. */ - if (_add_map_to_module(m)) { + if (add_map_to_module(m)) { goto except; } - + // if (PyType_Ready(&cPhysRecType)) { // return NULL; // } diff --git a/src/cpy/Object/cObject.c b/src/cpy/Object/cObject.c new file mode 100644 index 0000000..7f929bf --- /dev/null +++ b/src/cpy/Object/cObject.c @@ -0,0 +1,470 @@ +/* Use this file as a template to start implementing a module that + also declares object types. All occurrences of 'MyObj' should be changed + to something reasonable for your objects. After that, all other + occurrences of 'cObject' should be changed to something reasonable for your + module. If your module is named foo your sourcefile should be named + foomodule.c. + + You will probably want to delete all references to 'x_attr' and add + your own types of attributes instead. Maybe you want to name your + local variables other than 'self'. If your object type is needed in + other files, you'll have to create a file "foobarobject.h"; see + floatobject.h for an example. */ + +/* MyObj objects */ + +#include "Python.h" + +static PyObject *ErrorObject; + +typedef struct { + PyObject_HEAD + PyObject *x_attr; /* Attributes dictionary, NULL on construction, will be populated by MyObj_getattro. */ +} ObjectWithAttributes; + +/** Forward declaration. */ +static PyTypeObject ObjectWithAttributes_Type; + +#define ObjectWithAttributes_Check(v) (Py_TYPE(v) == &ObjectWithAttributes_Type) + +static ObjectWithAttributes * +ObjectWithAttributes_new(PyObject *Py_UNUSED(arg)) { + ObjectWithAttributes *self; + self = PyObject_New(ObjectWithAttributes, &ObjectWithAttributes_Type); + if (self == NULL) { + return NULL; + } + self->x_attr = NULL; + return self; +} + +/* ObjectWithAttributes methods */ +static void +ObjectWithAttributes_dealloc(ObjectWithAttributes *self) { + Py_XDECREF(self->x_attr); + PyObject_Del(self); +} + +static PyObject * +ObjectWithAttributes_demo(ObjectWithAttributes *Py_UNUSED(self), PyObject *args) { + if (!PyArg_ParseTuple(args, ":demo")) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef ObjectWithAttributes_methods[] = { + {"demo", (PyCFunction) ObjectWithAttributes_demo, METH_VARARGS, + PyDoc_STR("demo() -> None")}, + {NULL, NULL, 0, NULL} /* sentinel */ +}; + +static PyObject * +ObjectWithAttributes_getattro(ObjectWithAttributes *self, PyObject *name) { + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItem(self->x_attr, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + } + return PyObject_GenericGetAttr((PyObject *) self, name); +} + +static int +ObjectWithAttributes_setattr(ObjectWithAttributes *self, char *name, PyObject *v) { + if (self->x_attr == NULL) { + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) + return -1; + } + if (v == NULL) { + int rv = PyDict_DelItemString(self->x_attr, name); + if (rv < 0) + PyErr_SetString(PyExc_AttributeError, + "delete non-existing ObjectWithAttributes attribute"); + return rv; + } else + /* v is a borrowed reference, then PyDict_SetItemString() does NOT steal it so nothing to do. */ + return PyDict_SetItemString(self->x_attr, name, v); +} + +static PyTypeObject ObjectWithAttributes_Type = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "cObject.ObjectWithAttributes", /*tp_name*/ + sizeof(ObjectWithAttributes), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor) ObjectWithAttributes_dealloc, /*tp_dealloc*/ +#if PY_MINOR_VERSION < 8 + 0, /*tp_print*/ +#else + 0, /* Py_ssize_t tp_vectorcall_offset; */ +#endif + (getattrfunc) 0, /*tp_getattr*/ + (setattrfunc) ObjectWithAttributes_setattr, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + (getattrofunc) ObjectWithAttributes_getattro, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + ObjectWithAttributes_methods, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ +// PyType_GenericNew, /*tp_new*/ + (newfunc) ObjectWithAttributes_new, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + NULL, /* tp_bases */ + NULL, /* tp_mro */ + NULL, /* tp_cache */ + NULL, /* tp_subclasses */ + NULL, /* tp_weaklist */ + NULL, /* tp_del */ + 0, /* tp_version_tag */ + NULL, /* tp_finalize */ +#if PY_MINOR_VERSION > 7 + NULL, /* tp_vectorcall */ +#endif +#if PY_MINOR_VERSION == 8 + 0, /*tp_print*/ +#endif +#if PY_MINOR_VERSION >= 12 + '\0', /* unsigned char tp_watched */ +#if PY_MINOR_VERSION >= 13 + 0, /* uint16_t tp_versions_used */ +#endif +#endif +}; +/* --------------------------------------------------------------------- */ + +/* Function of two integers returning integer */ + +#if 0 +PyDoc_STRVAR(cObject_foo_doc, + "foo(i,j)\n\ +\n\ +Return the sum of i and j."); + +static PyObject * +cObject_foo(PyObject *Py_UNUSED(self), PyObject *args) { + long i, j; + long res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { + return NULL; + } + res = i + j; /* cObjX Do something here */ + return PyLong_FromLong(res); +} + + +/* Function of no arguments returning new MyObj object */ +static PyObject * +cObject_ObjectWithAttributes_new(PyObject *Py_UNUSED(self), PyObject *args) { + ObjectWithAttributes *rv; + + if (!PyArg_ParseTuple(args, ":new")) { + return NULL; + } + rv = ObjectWithAttributes_new(args); + if (rv == NULL) { + return NULL; + } + return (PyObject *) rv; +} + +/* Example with subtle bug from extensions manual ("Thin Ice"). */ +static PyObject * +cObject_thin_ice_bug(PyObject *Py_UNUSED(self), PyObject *args) { + PyObject *list, *item; + + if (!PyArg_ParseTuple(args, "O:bug", &list)) + return NULL; + + item = PyList_GetItem(list, 0); + /* Py_INCREF(item); */ + PyList_SetItem(list, 1, PyLong_FromLong(0L)); + PyObject_Print(item, stdout, 0); + printf("\n"); + /* Py_DECREF(item); */ + + Py_INCREF(Py_None); + return Py_None; +} + +/* Test bad format character */ +static PyObject * +cObject_roj(PyObject *Py_UNUSED(self), PyObject *args) { + PyObject *a; + long b; + if (!PyArg_ParseTuple(args, "O#:roj", &a, &b)) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} +#endif + +/* ---------- */ + +static PyTypeObject Str_Type = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "cObject.Str", /*tp_name*/ + 0, /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + 0, /*tp_dealloc*/ +#if PY_MINOR_VERSION < 8 + 0, /*tp_print*/ +#else + 0, /* Py_ssize_t tp_vectorcall_offset; */ +#endif + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /* see PyInit_cObject */ /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + NULL, /* tp_bases */ + NULL, /* tp_mro */ + NULL, /* tp_cache */ + NULL, /* tp_subclasses */ + NULL, /* tp_weaklist */ + NULL, /* tp_del */ + 0, /* tp_version_tag */ + NULL, /* tp_finalize */ +#if PY_MINOR_VERSION > 7 + NULL, /* tp_vectorcall */ +#endif +#if PY_MINOR_VERSION == 8 + 0, /*tp_print*/ +#endif +#if PY_MINOR_VERSION >= 12 + '\0', /* unsigned char tp_watched */ +#if PY_MINOR_VERSION >= 13 + 0, /* uint16_t tp_versions_used */ +#endif +#endif +}; + +/* ---------- */ + +static PyObject * +null_richcompare(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(other), int Py_UNUSED(op)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + +static PyTypeObject Null_Type = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "cObject.Null", /*tp_name*/ + 0, /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ +#if PY_MINOR_VERSION < 8 + 0, /*tp_print*/ +#else + 0, /* Py_ssize_t tp_vectorcall_offset; */ +#endif + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + null_richcompare, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /* see PyInit_cObject */ /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /* see PyInit_cObject */ /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + NULL, /* tp_bases */ + NULL, /* tp_mro */ + NULL, /* tp_cache */ + NULL, /* tp_subclasses */ + NULL, /* tp_weaklist */ + NULL, /* tp_del */ + 0, /* tp_version_tag */ + NULL, /* tp_finalize */ +#if PY_MINOR_VERSION > 7 + NULL, /* tp_vectorcall */ +#endif +#if PY_MINOR_VERSION == 8 + 0, /*tp_print*/ +#endif +#if PY_MINOR_VERSION >= 12 + '\0', /* unsigned char tp_watched */ +#if PY_MINOR_VERSION >= 13 + 0, /* uint16_t tp_versions_used */ +#endif +#endif +}; + + +/* ---------- */ + + +/* List of functions defined in the module */ +static PyMethodDef cObject_functions[] = { +#if 0 + {"roj", cObject_roj, METH_VARARGS, + PyDoc_STR("roj(a,b) -> None")}, + {"foo", cObject_foo, METH_VARARGS, + cObject_foo_doc}, + {"new_ObjectWithAttributes", cObject_ObjectWithAttributes_new, METH_VARARGS, + PyDoc_STR("new() -> new ObjectWithAttributes object")}, + {"thin_ice_bug", cObject_thin_ice_bug, METH_VARARGS, + PyDoc_STR("bug(o) -> None")}, +#endif + {NULL, NULL, 0, NULL} /* sentinel */ +}; + +PyDoc_STRVAR(module_doc, "This is a template module just for instruction."); + +/* Initialization function for the module (*must* be called PyInit_cObject) */ + + +static struct PyModuleDef cObject = { + PyModuleDef_HEAD_INIT, + "cObject", + module_doc, + -1, + cObject_functions, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit_cObject(void) { + PyObject *m = NULL; + + /* Due to cross-platform compiler issues the slots must be filled + * here. It's required for portability to Windows without requiring + * C++. */ + Null_Type.tp_base = &PyBaseObject_Type; + Null_Type.tp_new = PyType_GenericNew; + Str_Type.tp_base = &PyUnicode_Type; + + /* Create the module and add the functions */ + m = PyModule_Create(&cObject); + if (m == NULL) { + goto fail; + } + /* Add some symbolic constants to the module */ + if (ErrorObject == NULL) { + ErrorObject = PyErr_NewException("cObject.error", NULL, NULL); + if (ErrorObject == NULL) + goto fail; + } + Py_INCREF(ErrorObject); + if (PyModule_AddObject(m, "error", ErrorObject)) { + goto fail; + } + /* Finalize the type object including setting type of the new type + * object; doing it here is required for portability, too. */ + if (PyType_Ready(&ObjectWithAttributes_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "ObjectWithAttributes", (PyObject *) &ObjectWithAttributes_Type)) { + goto fail; + } + /* Add Str */ + if (PyType_Ready(&Str_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "Str", (PyObject *) &Str_Type)) { + goto fail; + } + /* Add Null */ + if (PyType_Ready(&Null_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "Null", (PyObject *) &Null_Type)) { + goto fail; + } + return m; + fail: + Py_XDECREF(m); + return NULL; +} diff --git a/src/cpy/Object/cSeqObject.c b/src/cpy/Object/cSeqObject.c new file mode 100644 index 0000000..5709a7c --- /dev/null +++ b/src/cpy/Object/cSeqObject.c @@ -0,0 +1,414 @@ +// +// Created by Paul Ross on 08/07/2021. +// +// Example of an object implementing the sequence methods with PySequenceMethods. +// +// See also src/cpy/Iterators/cIterator.c +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +typedef struct { + PyObject_HEAD + long *array_long; + ssize_t size; +} SequenceLongObject; + +static PyObject * +SequenceLongObject_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { + SequenceLongObject *self; + self = (SequenceLongObject *) type->tp_alloc(type, 0); + if (self != NULL) { + assert(!PyErr_Occurred()); + self->size = 0; + self->array_long = NULL; + } + return (PyObject *) self; +} + +static int +SequenceLongObject_init(SequenceLongObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sequence", NULL}; + PyObject *sequence = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &sequence)) { + return -1; + } + if (!PySequence_Check(sequence)) { + return -2; + } + self->size = PySequence_Length(sequence); + self->array_long = malloc(self->size * sizeof(long)); + if (!self->array_long) { + return -3; + } + for (Py_ssize_t i = 0; i < self->size; ++i) { + // New reference. + PyObject *py_value = PySequence_GetItem(sequence, i); + if (PyLong_Check(py_value)) { + self->array_long[i] = PyLong_AsLong(py_value); + Py_DECREF(py_value); + } else { + PyErr_Format( + PyExc_TypeError, + "Argument [%zd] must be a int, not type %s", + i, + Py_TYPE(sequence)->tp_name + ); + // Clean up on error. + free(self->array_long); + self->array_long = NULL; + Py_DECREF(py_value); + return -4; + } + } + return 0; +} + +static void +SequenceLongObject_dealloc(SequenceLongObject *self) { + free(self->array_long); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyMethodDef SequenceLongObject_methods[] = { +// { +// "size", +// (PyCFunction) SequenceLongObject_size, +// METH_NOARGS, +// "Return the size of the sequence." +// }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +/* Sequence methods. */ +static Py_ssize_t +SequenceLongObject_sq_length(PyObject *self) { +// fprintf(stdout, "%s(%p): returns=%zd\n", __FUNCTION__, (void *) self, ((SequenceLongObject *) self)->size); + return ((SequenceLongObject *) self)->size; +} + +// Forward references +static PyTypeObject SequenceLongObjectType; + +static int is_sequence_of_long_type(PyObject *op); + +/** + * Returns a new SequenceLongObject composed of self + other. + * @param self + * @param other + * @return + */ +static PyObject * +SequenceLongObject_sq_concat(PyObject *self, PyObject *other) { +// fprintf(stdout, "%s(%p):\n", __FUNCTION__, (void *) self); + if (!is_sequence_of_long_type(other)) { + PyErr_Format( + PyExc_TypeError, + "%s(): argument 1 must have type \"SequenceLongObject\" not %s", + Py_TYPE(other)->tp_name + ); + return NULL; + } + PyObject *ret = SequenceLongObject_new(&SequenceLongObjectType, NULL, NULL); + if (!ret) { + assert(PyErr_Occurred()); + return NULL; + } + /* For convenience. */ + SequenceLongObject *ret_as_slo = (SequenceLongObject *) ret; + ret_as_slo->size = ((SequenceLongObject *) self)->size + ((SequenceLongObject *) other)->size; + ret_as_slo->array_long = malloc(ret_as_slo->size * sizeof(long)); + if (!ret_as_slo->array_long) { + PyErr_Format(PyExc_MemoryError, "%s(): Can not create new object.", __FUNCTION__); + Py_DECREF(ret); + return NULL; + } +// fprintf(stdout, "%s(): New %p size=%zd\n", __FUNCTION__, (void *) ret_as_slo, ret_as_slo->size); + + ssize_t i = 0; + ssize_t ub = ((SequenceLongObject *) self)->size; + while (i < ub) { +// fprintf(stdout, "%s(): Setting from %p [%zd] to [%zd]\n", __FUNCTION__, (void *) self, i, i); + ret_as_slo->array_long[i] = ((SequenceLongObject *) self)->array_long[i]; + i++; + } + ssize_t j = 0; + ub = ((SequenceLongObject *) other)->size; + while (j < ub) { +// fprintf(stdout, "%s(): Setting %p [%zd] to [%zd]\n", __FUNCTION__, (void *) other, j, i); + ret_as_slo->array_long[i] = ((SequenceLongObject *) other)->array_long[j]; + i++; + j++; + } + return ret; +} + +/** + * Return a new sequence which contains the old one repeated count times. + * @param self + * @param count + * @return + */ +static PyObject * +SequenceLongObject_sq_repeat(PyObject *self, Py_ssize_t count) { + PyObject *ret = SequenceLongObject_new(&SequenceLongObjectType, NULL, NULL); + if (!ret) { + assert(PyErr_Occurred()); + return NULL; + } + assert(ret != self); + if (((SequenceLongObject *) self)->size > 0 && count > 0) { + /* For convenience. */ + SequenceLongObject *self_as_slo = (SequenceLongObject *) self; + SequenceLongObject *ret_as_slo = (SequenceLongObject *) ret; + ret_as_slo->size = self_as_slo->size * count; + assert(ret_as_slo->size > 0); + ret_as_slo->array_long = malloc(ret_as_slo->size * sizeof(long)); + if (!ret_as_slo->array_long) { + PyErr_Format(PyExc_MemoryError, "%s(): Can not create new object.", __FUNCTION__); + Py_DECREF(ret); + return NULL; + } + Py_ssize_t ret_index = 0; + for (Py_ssize_t i = 0; i < count; ++i) { +// fprintf(stdout, "%s(): Setting %p Count %zd\n", __FUNCTION__, (void *) ret, i); + for (Py_ssize_t j = 0; j < self_as_slo->size; ++j) { +// fprintf( +// stdout, "%s(): Setting %p [%zd] to %zd\n", +// __FUNCTION__, (void *) ret, ret_index, self_as_slo->array_long[j] +// ); + ret_as_slo->array_long[ret_index] = self_as_slo->array_long[j]; + ++ret_index; + } + } + } else { + /* Empty sequence. */ + } + return ret; +} + +/** + * Returns a new reference to an indexed item in a sequence. + * @param self + * @param index + * @return + */ +static PyObject * +SequenceLongObject_sq_item(PyObject *self, Py_ssize_t index) { +// fprintf(stdout, "%s(): index=%zd\n", __FUNCTION__, index); + Py_ssize_t my_index = index; + if (my_index < 0) { + my_index += SequenceLongObject_sq_length(self); + } + // Corner case example: len(self) == 0 and index < 0 + if (my_index < 0 || my_index >= SequenceLongObject_sq_length(self)) { +// fprintf(stdout, "%s(): index=%zd\n", __FUNCTION__, index); + PyErr_Format( + PyExc_IndexError, + "Index %ld is out of range for length %ld", + index, + SequenceLongObject_sq_length(self) + ); + return NULL; + } + return PyLong_FromLong(((SequenceLongObject *) self)->array_long[my_index]); +} + +static int +SequenceLongObject_sq_ass_item(PyObject *self, Py_ssize_t index, PyObject *value) { + fprintf( + stdout, "%s()#%d: self=%p index=%zd value=%p\n", + __FUNCTION__, __LINE__, (void *) self, index, (void *) value + ); + /* This is very weird. + * When the given index is negative and out of range PyObject_SetItem() + * and PyObject_DelItem() will have *already* added the sequence length + * before calling this function. + * So to get the original out of range negative index we have to *subtract* + * the sequence length. */ + if (index < 0) { + fprintf( + stdout, "%s()#%d: Fixing index index=%zd to %zd\n", __FUNCTION__, __LINE__, + index, index - SequenceLongObject_sq_length(self) + ); + index -= SequenceLongObject_sq_length(self); + } + /* Isn't it? */ + Py_ssize_t my_index = index; + if (my_index < 0) { + my_index += SequenceLongObject_sq_length(self); + } + // Corner case example: len(self) == 0 and index < 0 + fprintf( + stdout, "%s()#%d: len=%zd index=%zd my_index=%zd\n", __FUNCTION__, __LINE__, + SequenceLongObject_sq_length(self), index, my_index + ); + if (my_index < 0 || my_index >= SequenceLongObject_sq_length(self)) { + PyErr_Format( + PyExc_IndexError, + "Index %ld is out of range for length %ld", + index, + SequenceLongObject_sq_length(self) + ); + return -1; + } + if (value != NULL) { + /* Just set the value. */ + if (!PyLong_Check(value)) { + PyErr_Format( + PyExc_TypeError, + "sq_ass_item value needs to be an int, not type %s", + Py_TYPE(value)->tp_name + ); + return -1; + } + ((SequenceLongObject *) self)->array_long[my_index] = PyLong_AsLong(value); + } else { + /* Delete the value. */ + /* For convenience. */ + SequenceLongObject *self_as_slo = (SequenceLongObject *) self; + /* Special case: deleting the only item in the array. */ + if (self_as_slo->size == 1) { + fprintf(stdout, "%s()#%d: deleting empty index\n", __FUNCTION__, __LINE__); + free(self_as_slo->array_long); + self_as_slo->array_long = NULL; + self_as_slo->size = 0; + } else { + /* Delete the value and re-compose the array. */ + fprintf(stdout, "%s()#%d: deleting index=%zd\n", __FUNCTION__, __LINE__, index); + long *new_array = malloc((self_as_slo->size - 1) * sizeof(long)); + if (!new_array) { + PyErr_Format( + PyExc_MemoryError, + "sq_ass_item can not allocate new array. %s#%d", + __FILE__, __LINE__ + ); + return -1; + } + /* Copy up to the index. */ + Py_ssize_t index_new_array = 0; + for (Py_ssize_t i = 0; i < my_index; ++i, ++index_new_array) { + new_array[index_new_array] = self_as_slo->array_long[i]; + } + /* Copy past the index. */ + for (Py_ssize_t i = my_index + 1; i < self_as_slo->size; ++i, ++index_new_array) { + new_array[index_new_array] = self_as_slo->array_long[i]; + } + + free(self_as_slo->array_long); + self_as_slo->array_long = new_array; + --self_as_slo->size; + } + } + return 0; +} + +/** + * If an item in self is equal to value, return 1, otherwise return 0. On error, return -1. + * @param self + * @param value + * @return + */ +static int +SequenceLongObject_sq_contains(PyObject *self, PyObject *value) { + fprintf( + stdout, "%s()#%d: self=%p value=%p\n", + __FUNCTION__, __LINE__, (void *) self, (void *) value + ); + if (!PyLong_Check(value)) { + /* Alternates: Could raise TypeError or return -1. + * Here we act benignly! */ + return 0; + } + long c_value = PyLong_AsLong(value); + /* For convenience. */ + SequenceLongObject *self_as_slo = (SequenceLongObject *) self; + for (Py_ssize_t i = 0; i < SequenceLongObject_sq_length(self); ++i) { + if (self_as_slo->array_long[i] == c_value) { + return 1; + } + } + return 0; +} + +static PySequenceMethods SequenceLongObject_sequence_methods = { + .sq_length = (lenfunc)SequenceLongObject_sq_length, + .sq_concat = (binaryfunc)SequenceLongObject_sq_concat, + .sq_repeat = (ssizeargfunc)SequenceLongObject_sq_repeat, + .sq_item = (ssizeargfunc)SequenceLongObject_sq_item, + .sq_ass_item = (ssizeobjargproc)SequenceLongObject_sq_ass_item, + .sq_contains = (objobjproc)SequenceLongObject_sq_contains, + .sq_inplace_concat = (binaryfunc)NULL, + .sq_inplace_repeat = (ssizeargfunc)NULL, +}; + +static PyObject * +SequenceLongObject___str__(SequenceLongObject *self, PyObject *Py_UNUSED(ignored)) { + assert(!PyErr_Occurred()); + return PyUnicode_FromFormat("", self->size); +} + +static PyTypeObject SequenceLongObjectType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "SequenceLongObject", + .tp_basicsize = sizeof(SequenceLongObject), + .tp_itemsize = 0, + .tp_dealloc = (destructor) SequenceLongObject_dealloc, + .tp_as_sequence = &SequenceLongObject_sequence_methods, + .tp_str = (reprfunc) SequenceLongObject___str__, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "Sequence of long integers.", +// .tp_iter = NULL, +// .tp_iternext = NULL, + .tp_methods = SequenceLongObject_methods, + .tp_init = (initproc) SequenceLongObject_init, + .tp_new = SequenceLongObject_new, +}; + +static int +is_sequence_of_long_type(PyObject *op) { + return Py_TYPE(op) == &SequenceLongObjectType; +} + +static PyMethodDef cIterator_methods[] = { +// {"iterate_and_print", (PyCFunction) iterate_and_print, METH_VARARGS, +// "Iteratee through the argument printing the values."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef sequence_object_cmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cSeqObject", + .m_doc = ( + "Example module that creates an extension type with sequence methods" + ), + .m_size = -1, + .m_methods = cIterator_methods, +}; + +PyMODINIT_FUNC +PyInit_cSeqObject(void) { + PyObject *m; + m = PyModule_Create(&sequence_object_cmodule); + if (m == NULL) { + return NULL; + } + + if (PyType_Ready(&SequenceLongObjectType) < 0) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(&SequenceLongObjectType); + if (PyModule_AddObject( + m, + "SequenceLongObject", + (PyObject *) &SequenceLongObjectType) < 0 + ) { + Py_DECREF(&SequenceLongObjectType); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/ParseArgs/cParseArgs.c b/src/cpy/ParseArgs/cParseArgs.c new file mode 100644 index 0000000..0ae1c61 --- /dev/null +++ b/src/cpy/ParseArgs/cParseArgs.c @@ -0,0 +1,453 @@ +// +// cParseArgs.c +// PythonExtensionPatterns +// +// Created by Paul Ross on 08/05/2014. +// Copyright (c) 2014 Paul Ross. All rights reserved. +// + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +#include "time.h" + +#define FPRINTF_DEBUG 0 + +/****************** Parsing arguments. ****************/ +static PyObject *parse_no_args(PyObject *Py_UNUSED(module)) { +#if FPRINTF_DEBUG + PyObject_Print(module, stdout, 0); + fprintf(stdout, "\nparse_no_args()\n"); +#endif + Py_RETURN_NONE; +} + +static PyObject *parse_one_arg(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) { +#if FPRINTF_DEBUG + PyObject_Print(module, stdout, 0); + fprintf(stdout, "\nparse_one_arg(): "); + PyObject_Print(arg, stdout, 0); + fprintf(stdout, "\n"); +#endif + /* Your code here...*/ + Py_RETURN_NONE; +} + +/** Example of a METH_VARGS function that takes a bytes object and int and an optional string. + * Returns the number of arguments parsed. + * + * Signature is: + * + * def parse_args(a: bytes, b: int, c: str = 'default_string') -> typing.Tuple[bytes, int, str]: + * */ +static PyObject *parse_args(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *arg_0 = NULL; + int arg_1; + char *arg_2 = "default_string"; + +#if FPRINTF_DEBUG + PyObject_Print(module, stdout, 0); + fprintf(stdout, "\nparse_args(): "); + PyObject_Print(args, stdout, 0); + fprintf(stdout, "\n"); +#endif + + if (!PyArg_ParseTuple(args, "Si|s", &arg_0, &arg_1, &arg_2)) { + return NULL; + } + /* Your code here...*/ + +// /* PyTuple_Size returns a Py_ssize_t */ +// return Py_BuildValue("n", PyTuple_Size(args)); + return Py_BuildValue("Ois", arg_0, arg_1, arg_2); +} + + +/** This takes a Python object, 'sequence', that supports the sequence protocol and, optionally, an integer, 'count'. + * This returns a new sequence which is the old sequence multiplied by the count. + * + * def parse_args_kwargs(sequence=typing.Sequence[typing.Any], count: int = 1) -> typing.Sequence[typing.Any]: + * + * NOTE: If count is absent entirely then an empty sequence of given type is returned as count is assumed zero as + * optional. + * */ +static PyObject * +parse_args_kwargs(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + PyObject *ret = NULL; + PyObject *py_sequence = NULL; + int count = 1; /* Default. */ + static char *kwlist[] = { + "sequence", /* A sequence object, str, list, tuple etc. */ + "count", /* Python int converted to a C int. */ + NULL, + }; + +#if FPRINTF_DEBUG + PyObject_Print(module, stdout, 0); + fprintf(stdout, "\n"); + PyObject_Print(args, stdout, 0); + fprintf(stdout, "\n"); + PyObject_Print(kwargs, stdout, 0); + fprintf(stdout, "\n"); +#endif + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i", kwlist, &py_sequence, &count)) { + goto except; + } + + /* Your code here...*/ + ret = PySequence_Repeat(py_sequence, count); + if (ret == NULL) { + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; +} + +/** Parse the args where we are simulating immutable defaults of a string and a tuple. + * The defaults are: "Hello world", ("Answer", 42) + * + * This returns both arguments as a tuple. + * + * This imitates the Python way of handling defaults. + */ +static PyObject *parse_args_with_immutable_defaults(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the default arguments, initialised below. */ + static PyObject *pyObjDefaultArg_0; + static PyObject *pyObjDefaultArg_1; + /* These pointers are the ones we use in the body of the function, they + * either point at the supplied argument or the default (static) argument. + * We treat these as "borrowed" references and so incref and decref them + * appropriately. + */ + PyObject *pyObjArg_0 = NULL; + PyObject *pyObjArg_1 = NULL; + int have_inc_ref_arguments = 0; + + /* Set defaults for arguments. */ + if (!pyObjDefaultArg_0) { + pyObjDefaultArg_0 = PyUnicode_FromString("Hello world"); + if (!pyObjDefaultArg_0) { + PyErr_SetString(PyExc_RuntimeError, "Can not create string!"); + goto except; + } + } + if (!pyObjDefaultArg_1) { + pyObjDefaultArg_1 = PyTuple_New(2); + if (!pyObjDefaultArg_1) { + PyErr_SetString(PyExc_RuntimeError, "Can not create tuple!"); + goto except; + } + if (PyTuple_SetItem(pyObjDefaultArg_1, 0, PyUnicode_FromString("Answer"))) { + PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[0]!"); + goto except; + } + if (PyTuple_SetItem(pyObjDefaultArg_1, 1, PyLong_FromLong(42))) { + PyErr_SetString(PyExc_RuntimeError, "Can not set tuple[1]!"); + goto except; + } + } + + if (!PyArg_ParseTuple(args, "|OO", &pyObjArg_0, &pyObjArg_1)) { + goto except; + } + /* If optional arguments absent then switch to defaults. */ + if (!pyObjArg_0) { + pyObjArg_0 = pyObjDefaultArg_0; + } + /* Borrowed reference. */ + Py_INCREF(pyObjArg_0); + if (!pyObjArg_1) { + pyObjArg_1 = pyObjDefaultArg_1; + } + /* Borrowed reference. */ + Py_INCREF(pyObjArg_1); + have_inc_ref_arguments = 1; + +#if FPRINTF_DEBUG + fprintf(stdout, "pyObjArg0 was: "); + PyObject_Print(pyObjArg_0, stdout, 0); + fprintf(stdout, "\n"); + fprintf(stdout, "pyObjArg1 was: "); + PyObject_Print(pyObjArg_1, stdout, 0); + fprintf(stdout, "\n"); +#endif + + /* Your code here...*/ + + /* In this case we just return the arguments so we keep the incremented (borrowed) references. + * If the were not being returned then the borrowed references must be decremented: + * Py_XDECREF(pyObjArg_0); + * Py_XDECREF(pyObjArg_1); + * Here we use have_inc_ref_arguments to decide if we are entering except with incremented borrowed references. + */ + ret = Py_BuildValue("OO", pyObjArg_0, pyObjArg_1); + if (ret == NULL) { + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + /* Error, so decrement borrowed references if have_inc_ref_arguments. */ + if (have_inc_ref_arguments) { + Py_XDECREF(pyObjArg_0); + Py_XDECREF(pyObjArg_1); + } + ret = NULL; + finally: + return ret; +} + +/** Parse the args where we are simulating mutable default of an empty list. + * + * This is equivalent to: + * + * def parse_args_with_mutable_defaults_macro_helper(obj, default_list=[]): + * default_list.append(obj) + * return default_list + * + * See also parse_args_with_mutable_defaults_macro_helper() in cParseArgsHelper.cpp + * This adds the object to the list and returns None. + * + * This imitates the Python way of handling defaults. + */ +static PyObject *parse_args_with_mutable_defaults(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + /* Pointers to the default argument, initialised below. */ + static PyObject *arg_1_default = NULL; + /* Set defaults for argument 1. */ + if (!arg_1_default) { + arg_1_default = PyList_New(0); + } + /* This pointer is the one we use in the body of the function, it + * either points at the supplied argument or the default (static) argument. + */ + PyObject *arg_1 = NULL; + + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &arg_1)) { + goto except; + } + /* If optional argument absent then switch to defaults. */ + if (!arg_1) { + arg_1 = arg_1_default; + } + +#if FPRINTF_DEBUG + fprintf(stdout, "pyObjArg1 was: "); + PyObject_Print(pyObjArg_1, stdout, 0); + fprintf(stdout, "\n"); +#endif + + /* Your code here...*/ + + /* Append the first argument to the second. + * PyList_Append increments the reference count of arg_0. */ + if (PyList_Append(arg_1, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; + } + +#if FPRINTF_DEBUG + fprintf(stdout, "pyObjArg1 now: "); + PyObject_Print(pyObjArg_1, stdout, 0); + fprintf(stdout, "\n"); +#endif + + /* Success. */ + assert(!PyErr_Occurred()); + /* Increments the default or the given argument. */ + Py_INCREF(arg_1); + ret = arg_1; + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; +finally: + return ret; +} + +/** + * Example of setting a bytes default argument. + * + * Signature: + * + * def parse_default_bytes_object(b: bytes = b"default") -> bytes: + */ +static PyObject * +parse_default_bytes_object(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + static const char *arg_default = "default"; + Py_buffer arg; + arg.buf = (void *) arg_default; + arg.len = strlen(arg_default); + static char *kwlist[] = { + "b", + NULL, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|y*", kwlist, &arg)) { + assert(PyErr_Occurred()); + return NULL; + } + return Py_BuildValue("y#", arg.buf, arg.len); +} + +/** Positional only and keyword only arguments. + * + * Reproducing https://docs.python.org/3/tutorial/controlflow.html#special-parameters + * + * Equivalent to the Python function: + * + * def parse_pos_only_kwd_only(pos1: str, pos2: int, /, pos_or_kwd: bytes, *, kwd1: float, kwd2: int) -> typing.Tuple[typing.Any, ...]: + * return None + * */ +static PyObject * +parse_pos_only_kwd_only(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + /* Arguments, first three are required. */ + Py_buffer pos1; + int pos2; + Py_buffer pos_or_kwd; + /* Last two are optional. */ + double kwd1 = 256.0; + int kwd2 = -421; + static char *kwlist[] = { + "", /* pos1 is positional only. */ + "", /* pos2 is positional only. */ + "pos_or_kwd", /* pos_or_kwd can be positional or keyword argument. */ + "kwd1", /* kwd1 is keyword only argument by use of '$' in format string. */ + "kwd2", /* kwd2 is keyword only argument by use of '$' in format string. */ + NULL, + }; + +#if FPRINTF_DEBUG + // PyObject_Print(module, stdout, 0); + fprintf(stdout, "parse_pos_only_kwd_only():\n"); + PyObject_Print(args, stdout, 0); + fprintf(stdout, "\n"); + PyObject_Print(kwargs, stdout, 0); + fprintf(stdout, "\n"); +#endif + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*iy*|$di", kwlist, &pos1, &pos2, &pos_or_kwd, &kwd1, &kwd2)) { + assert(PyErr_Occurred()); + return NULL; + } + /* Return the parsed arguments. */ + return Py_BuildValue("s#iy#di", pos1.buf, pos1.len, pos2, pos_or_kwd.buf, pos_or_kwd.len, kwd1, kwd2); +} + +/** Checks that a list is full of Python integers. */ +int sum_list_of_longs(PyObject *list_longs, void *address) { + PyObject *item = NULL; + + if (!list_longs || !PyList_Check(list_longs)) { /* Note: PyList_Check allows sub-types. */ + PyErr_Format(PyExc_TypeError, "check_list_of_longs(): First argument is not a list"); + return 0; + } + long result = 0L; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list_longs); ++i) { + item = PyList_GetItem(list_longs, i); + if (!PyLong_CheckExact(item)) { + PyErr_Format(PyExc_TypeError, "check_list_of_longs(): Item %d is not a Python integer.", i); + return 0; + } + /* PyLong_AsLong() must always succeed because of check above. */ + result += PyLong_AsLong(item); + } + long *p_long = (long *) address; + *p_long = result; + return 1; /* Success. */ +} + +/** Parse the args where we are expecting a single argument that must be a + * list of numbers. + * + * This returns the sum of the numbers as a Python integer. + * + * This illustrates the use of "O&" in parsing. + */ +static PyObject * +parse_args_with_function_conversion_to_c(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *ret = NULL; + long result; + +#if FPRINTF_DEBUG + PyObject_Print(module, stdout, 0); + fprintf(stdout, "\n"); + PyObject_Print(args, stdout, 0); + fprintf(stdout, "\n"); +#endif + + if (!PyArg_ParseTuple(args, "O&", sum_list_of_longs, &result)) { + /* NOTE: If check_list_of_numbers() returns 0 an error should be set. */ + assert(PyErr_Occurred()); + goto except; + } + + /* Your code here...*/ + ret = PyLong_FromLong(result); + if (ret == NULL) { + goto except; + } + assert(!PyErr_Occurred()); + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; +} + + +static char parse_args_kwargs_docstring[] = + "Some documentation for this function."; + +static PyMethodDef cParseArgs_methods[] = { + {"parse_no_args", (PyCFunction) parse_no_args, METH_NOARGS, "No arguments."}, + {"parse_one_arg", (PyCFunction) parse_one_arg, METH_O, "One argument."}, + {"parse_args", (PyCFunction) parse_args, METH_VARARGS, "Reads args only."}, + {"parse_args_kwargs", (PyCFunction) parse_args_kwargs, METH_VARARGS | + METH_KEYWORDS, parse_args_kwargs_docstring}, + {"parse_args_with_immutable_defaults", (PyCFunction) parse_args_with_immutable_defaults, + METH_VARARGS, "A function with mutable defaults."}, + {"parse_args_with_mutable_defaults", (PyCFunction) parse_args_with_mutable_defaults, + METH_VARARGS, "A function with mutable defaults."}, + {"parse_default_bytes_object", (PyCFunction) parse_default_bytes_object, METH_VARARGS | + METH_KEYWORDS, "Example of default bytes object."}, + {"parse_pos_only_kwd_only", (PyCFunction) parse_pos_only_kwd_only, METH_VARARGS | + METH_KEYWORDS, "Positional and keyword only arguments"}, + {"parse_args_with_function_conversion_to_c", (PyCFunction) parse_args_with_function_conversion_to_c, METH_VARARGS, + "Parsing an argument that must be a list of numbers."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cParseArgs_module = { + PyModuleDef_HEAD_INIT, + "cParseArgs", + "Examples of parsing arguments in a Python 'C' extension.", + -1, + cParseArgs_methods, + NULL, /* inquiry m_reload */ + NULL, /* traverseproc m_traverse */ + NULL, /* inquiry m_clear */ + NULL, /* freefunc m_free */ +}; + +PyMODINIT_FUNC PyInit_cParseArgs(void) { + return PyModule_Create(&cParseArgs_module); +} +/****************** END: Parsing arguments. ****************/ diff --git a/src/cpy/ParseArgs/cParseArgsHelper.cpp b/src/cpy/ParseArgs/cParseArgsHelper.cpp new file mode 100644 index 0000000..ea5125f --- /dev/null +++ b/src/cpy/ParseArgs/cParseArgsHelper.cpp @@ -0,0 +1,428 @@ +// +// cParseArgsHelper.cpp +// PythonExtensionPatterns +// +// Created by Paul Ross on 07/07/2024. +// Copyright (c) 2024 Paul Ross. All rights reserved. +// +// NOTE: For some reason when reformatting this file as a *.cpp file the formatting goes horribly wrong. +// The solution is to comment out #include "Python.h" before reformatting. + +// NOTE: This is legacy code. +// In version 0.1.0 of this project the documentation described some helper techniques for handling default arguments. +// The section titles were "Simplifying Macros" and "Simplifying C++11 class'. +// The code samples in the document were incorrect, this is the correct version(s) of that concept. +// However these helper techniques don't really help very much as the defaults have to be PyObjects and +// subsequently converted to C types. +// This just introduces another layer of abstraction for no real gain. +// For those reasons the documentation sections were removed from version 0.2.0. +// This code remains incase it is of interest. + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" + +const long DEFAULT_ID = 1024L; +const double DEFAULT_FLOAT = 8.0; + +/****************** Parsing arguments. ****************/ + + +/* Helper macros. */ +#define PY_DEFAULT_ARGUMENT_INIT(name, value, ret) \ + PyObject *name = NULL; \ + static PyObject *default_##name = NULL; \ + if (! default_##name) { \ + default_##name = value; \ + if (! default_##name) { \ + PyErr_SetString( \ + PyExc_RuntimeError, \ + "Can not create default value for " #name \ + ); \ + return ret; \ + } \ + } + +#define PY_DEFAULT_ARGUMENT_SET(name) \ + if (! name) { \ + name = default_##name; \ + } + +#define PY_DEFAULT_CHECK(name, check_function, type) \ + if (!check_function(name)) { \ + PyErr_Format( \ + PyExc_TypeError, \ + #name " must be " #type ", not \"%s\"", \ + Py_TYPE(name)->tp_name \ + ); \ + return NULL; \ + } + + +static PyObject * +parse_defaults_with_helper_macro(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + PyObject *ret = NULL; + /* Initialise default arguments. Note: these might cause an early return. */ + PY_DEFAULT_ARGUMENT_INIT(encoding_m, PyUnicode_FromString("utf-8"), NULL); + PY_DEFAULT_ARGUMENT_INIT(the_id_m, PyLong_FromLong(DEFAULT_ID), NULL); + PY_DEFAULT_ARGUMENT_INIT(log_interval_m, PyFloat_FromDouble(DEFAULT_FLOAT), NULL); + + fprintf(stdout, "%s(): %s#%d", __FUNCTION__, __FILE_NAME__, __LINE__); + fprintf(stdout, " default_encoding_m %p", (void *)default_encoding_m); + if (default_encoding_m) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(default_encoding_m)); + } + fprintf(stdout, " encoding_m %p", (void *)encoding_m); + if (encoding_m) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_m)); + } + fprintf(stdout, "\n"); + + static const char *kwlist[] = {"encoding", "the_id", "log_interval", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", + const_cast(kwlist), + &encoding_m, &the_id_m, &log_interval_m)) { + goto except; + } + /* + * Assign absent arguments to defaults and increment the reference count. + * Don't forget to decrement the reference count before returning! + */ + PY_DEFAULT_ARGUMENT_SET(encoding_m); + PY_DEFAULT_ARGUMENT_SET(the_id_m); + PY_DEFAULT_ARGUMENT_SET(log_interval_m); + + fprintf(stdout, "%s(): %s#%d", __FUNCTION__, __FILE_NAME__, __LINE__); + fprintf(stdout, " default_encoding_m %p", (void *)default_encoding_m); + if (default_encoding_m) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(default_encoding_m)); + } + fprintf(stdout, " encoding_m %p", (void *)encoding_m); + if (encoding_m) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_m)); + } + fprintf(stdout, "\n"); + + /* Check the types of the given or default arguments. */ + PY_DEFAULT_CHECK(encoding_m, PyUnicode_Check, "str"); + PY_DEFAULT_CHECK(the_id_m, PyLong_Check, "int"); + PY_DEFAULT_CHECK(log_interval_m, PyFloat_Check, "float"); + + /* + * Use 'encoding': Python str, 'the_id': C long, 'must_log': C long from here on... + */ + + /* Py_BuildValue("O") increments the reference count. */ + ret = Py_BuildValue("OOO", encoding_m, the_id_m, log_interval_m); + + fprintf(stdout, "%s(): %s#%d", __FUNCTION__, __FILE_NAME__, __LINE__); + fprintf(stdout, " default_encoding_m %p", (void *)default_encoding_m); + if (default_encoding_m) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(default_encoding_m)); + } + fprintf(stdout, " encoding_m %p", (void *)encoding_m); + if (encoding_m) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_m)); + } + fprintf(stdout, "\n"); + + assert(!PyErr_Occurred()); + assert(ret); + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; +finally: +// Py_DECREF(encoding_m); +// Py_DECREF(the_id_m); +// Py_DECREF(log_interval_m); + return ret; +} + +/** Parse the args where we are simulating mutable default of an empty list. + * This uses the helper macros. + * + * This is equivalent to: + * + * def parse_mutable_defaults_with_helper_macro(obj, default_list=[]): + * default_list.append(obj) + * return default_list + * + * This adds the object to the list and returns None. + * + * This imitates the Python way of handling defaults. + */ +static PyObject *parse_mutable_defaults_with_helper_macro(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + /* Pointers to the default argument, initialised below. */ + /* Initialise default arguments. Note: these might cause an early return. */ + PY_DEFAULT_ARGUMENT_INIT(list_argument_m, PyList_New(0), NULL); + + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &list_argument_m)) { + goto except; + } + /* If optional argument absent then switch to defaults. */ + PY_DEFAULT_ARGUMENT_SET(list_argument_m); + PY_DEFAULT_CHECK(list_argument_m, PyList_Check, "list"); + + /* Your code here...*/ + + /* Append the first argument to the second. + * PyList_Append() increments the refcount of arg_0. */ + if (PyList_Append(list_argument_m, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; + } + + /* Success. */ + assert(!PyErr_Occurred()); + /* This increments the default or the given argument. */ + Py_INCREF(list_argument_m); + ret = list_argument_m; + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; +finally: + return ret; +} + +/* Helper classes. */ + +/** Class to simplify default arguments. + * + * Usage: + * + * static DefaultArg arg_0(PyLong_FromLong(1L)); + * static DefaultArg arg_1(PyUnicode_FromString("Default string.")); + * if (! arg_0 || ! arg_1) { + * return NULL; + * } + * + * if (! PyArg_ParseTupleAndKeywords(args, kwargs, "...", + const_cast(kwlist), + &arg_0, &arg_1, ...)) { + return NULL; + } + * + * Then just use arg_0, arg_1 as if they were a PyObject* (possibly + * might need to be cast to some specific PyObject*). + * + * WARN: This class is designed to be statically allocated. If allocated + * on the heap or stack it will leak memory. That could be fixed by + * implementing: + * + * ~DefaultArg() { Py_XDECREF(m_default); } + * + * But this will be highly dangerous when statically allocated as the + * destructor will be invoked with the Python interpreter in an + * uncertain state and will, most likely, segfault: + * "Python(39158,0x7fff78b66310) malloc: *** error for object 0x100511300: pointer being freed was not allocated" + */ +class DefaultArg { +public: + DefaultArg(PyObject *new_ref) : m_arg(NULL), m_default(new_ref) {} + + /// Allow setting of the (optional) argument with + /// PyArg_ParseTupleAndKeywords + PyObject **operator&() { + m_arg = NULL; + return &m_arg; + } + + /// Access the argument or the default if default. + operator PyObject *() const { + return m_arg ? m_arg : m_default; + } + + PyObject *obj() const { + return m_arg ? m_arg : m_default; + } + + /// Test if constructed successfully from the new reference. + explicit operator bool() { return m_default != NULL; } + + PyObject *arg() const { + return m_arg; + } + + PyObject *default_arg() const { + return m_default; + } +protected: + PyObject *m_arg; + PyObject *m_default; +}; + +static PyObject * +parse_defaults_with_helper_class(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + PyObject *ret = NULL; + /* Initialise default arguments. */ + static DefaultArg encoding_c(PyUnicode_FromString("utf-8")); + static DefaultArg the_id_c(PyLong_FromLong(DEFAULT_ID)); + static DefaultArg log_interval_c(PyFloat_FromDouble(DEFAULT_FLOAT)); + + fprintf(stdout, "%s(): %s#%d", __FUNCTION__, __FILE_NAME__, __LINE__); + fprintf(stdout, " default_encoding_c %p", (void *)encoding_c.default_arg()); + if (encoding_c.default_arg()) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_c.default_arg())); + } + fprintf(stdout, " encoding_c %p", (void *)encoding_c.arg()); + if (encoding_c.arg()) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_c.arg())); + } + fprintf(stdout, "\n"); + + /* Check that the defaults are non-NULL i.e. succesful. */ + if (!encoding_c || !the_id_c || !log_interval_c) { + return NULL; + } + + static const char *kwlist[] = {"encoding", "the_id", "log_interval", NULL}; + /* &encoding etc. accesses &m_arg in DefaultArg because of PyObject **operator&() */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", + const_cast(kwlist), + &encoding_c, &the_id_c, &log_interval_c)) { + return NULL; + } + + fprintf(stdout, "%s(): %s#%d", __FUNCTION__, __FILE_NAME__, __LINE__); + fprintf(stdout, " default_encoding_c %p", (void *)encoding_c.default_arg()); + if (encoding_c.default_arg()) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_c.default_arg())); + } + fprintf(stdout, " encoding_c %p", (void *)encoding_c.arg()); + if (encoding_c.arg()) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_c.arg())); + } + fprintf(stdout, "\n"); + + PY_DEFAULT_CHECK(encoding_c, PyUnicode_Check, "str"); + PY_DEFAULT_CHECK(the_id_c, PyLong_Check, "int"); + PY_DEFAULT_CHECK(log_interval_c, PyFloat_Check, "float"); + + /* + * Use encoding, the_id, must_log from here on as PyObject* since we have + * operator PyObject*() const ... + * + * So if we have a function: + * set_encoding(PyObject *obj) { ... } + */ +// set_encoding(encoding); + /* ... */ + + /* Py_BuildValue("O") increments the reference count. */ + ret = Py_BuildValue("OOO", encoding_c.obj(), the_id_c.obj(), log_interval_c.obj()); + + fprintf(stdout, "%s(): %s#%d", __FUNCTION__, __FILE_NAME__, __LINE__); + fprintf(stdout, " default_encoding_c %p", (void *)encoding_c.default_arg()); + if (encoding_c.default_arg()) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_c.default_arg())); + } + fprintf(stdout, " encoding_c %p", (void *)encoding_c.arg()); + if (encoding_c.arg()) { + fprintf(stdout, " refcount: %zd", Py_REFCNT(encoding_c.arg())); + } + fprintf(stdout, "\n"); + + return ret; +} + + +/** Parse the args where we are simulating mutable default of an empty list. + * This uses the helper class. + * + * This is equivalent to: + * + * def parse_mutable_defaults_with_helper_class(obj, default_list=[]): + * default_list.append(obj) + * return default_list + * + * This adds the object to the list and returns None. + * + * This imitates the Python way of handling defaults. + */ +static PyObject *parse_mutable_defaults_with_helper_class(PyObject *Py_UNUSED(module), + PyObject *args) { + PyObject *ret = NULL; + /* Pointers to the non-default argument, initialised by PyArg_ParseTuple below. */ + PyObject *arg_0 = NULL; + static DefaultArg list_argument_c(PyList_New(0)); + + if (!PyArg_ParseTuple(args, "O|O", &arg_0, &list_argument_c)) { + goto except; + } + PY_DEFAULT_CHECK(list_argument_c, PyList_Check, "list"); + + /* Your code here...*/ + + /* Append the first argument to the second. + * PyList_Append() increments the refcount of arg_0. */ + if (PyList_Append(list_argument_c, arg_0)) { + PyErr_SetString(PyExc_RuntimeError, "Can not append to list!"); + goto except; + } + + /* Success. */ + assert(!PyErr_Occurred()); + /* This increments the default or the given argument. */ + Py_INCREF(list_argument_c); + ret = list_argument_c; + goto finally; + except: + assert(PyErr_Occurred()); + Py_XDECREF(ret); + ret = NULL; + finally: + return ret; +} + +static PyMethodDef cParseArgsHelper_methods[] = { + { + "parse_defaults_with_helper_macro", + (PyCFunction) parse_defaults_with_helper_macro, + METH_VARARGS, + "A function with immutable defaults." + }, + { + "parse_mutable_defaults_with_helper_macro", + (PyCFunction) parse_mutable_defaults_with_helper_macro, + METH_VARARGS, + "A function with a mutable argument." + }, + { + "parse_defaults_with_helper_class", + (PyCFunction) parse_defaults_with_helper_class, + METH_VARARGS, + "A function with immutable defaults." + }, + { + "parse_mutable_defaults_with_helper_class", + (PyCFunction) parse_mutable_defaults_with_helper_class, + METH_VARARGS, + "A function with a mutable argument." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cParseArgsHelper_module = { + PyModuleDef_HEAD_INIT, + "cParseArgsHelper", + "Examples of helper macros and classes when parsing arguments in a Python C/C++ extension.", + -1, + cParseArgsHelper_methods, + NULL, /* inquiry m_reload */ + NULL, /* traverseproc m_traverse */ + NULL, /* inquiry m_clear */ + NULL, /* freefunc m_free */ +}; + +PyMODINIT_FUNC PyInit_cParseArgsHelper(void) { + return PyModule_Create(&cParseArgsHelper_module); +} +/****************** END: Parsing arguments. ****************/ diff --git a/src/cpy/Pickle/cCustomPickle.c b/src/cpy/Pickle/cCustomPickle.c new file mode 100644 index 0000000..435ec2e --- /dev/null +++ b/src/cpy/Pickle/cCustomPickle.c @@ -0,0 +1,302 @@ +// +// custom_pickle.c +// PythonExtensionPatterns +// +// Created by Paul Ross on 08/04/2021. +// Copyright (c) 2021 Paul Ross. All rights reserved. +// +// This adds pickling to a standard custom object. + +#define PY_SSIZE_T_CLEAN +#include +#include "structmember.h" + +#define FPRINTF_DEBUG 0 + +typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; +} CustomObject; + +static void +Custom_dealloc(CustomObject *self) +{ + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +Custom_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) +{ + CustomObject *self; + self = (CustomObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->first = PyUnicode_FromString(""); + if (self->first == NULL) { + Py_DECREF(self); + return NULL; + } + self->last = PyUnicode_FromString(""); + if (self->last == NULL) { + Py_DECREF(self); + return NULL; + } + self->number = 0; + } +#if FPRINTF_DEBUG + fprintf(stdout, "Custom_new() reference counts first %zu last %zu\n", Py_REFCNT(self->first), Py_REFCNT(self->last)); +#endif + return (PyObject *) self; +} + +static int +Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_XDECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_XDECREF(tmp); + } + return 0; +} + +static PyMemberDef Custom_members[] = { + {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, + "first name"}, + {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, + "last name"}, + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL, 0, 0, 0, NULL} /* Sentinel */ +}; + +static PyObject * +Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (self->first == NULL) { + PyErr_SetString(PyExc_AttributeError, "first"); + return NULL; + } + if (self->last == NULL) { + PyErr_SetString(PyExc_AttributeError, "last"); + return NULL; + } + return PyUnicode_FromFormat("%S %S", self->first, self->last); +} + +/* Pickle the object */ +static const char* PICKLE_VERSION_KEY = "_pickle_version"; +static int PICKLE_VERSION = 1; + +static PyObject * +Custom___getstate__(CustomObject *self, PyObject *Py_UNUSED(ignored)) { + PyObject *ret = Py_BuildValue("{sOsOsisi}", + "first", self->first, + "last", self->last, + "number", self->number, + PICKLE_VERSION_KEY, PICKLE_VERSION); +#if FPRINTF_DEBUG + fprintf(stdout, "Custom___getstate__ returning type %s\n", Py_TYPE(ret)->tp_name); +#endif + return ret; +} + +static PyObject * +Custom___setstate__(CustomObject *self, PyObject *state) { +#if FPRINTF_DEBUG + fprintf(stdout, "Custom___getstate__ getting type %s\n", Py_TYPE(state)->tp_name); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(state, &pos, &key, &value)) { + /* do something interesting with the values... */ + fprintf(stdout, "Types Key: %s Value: %s\n", Py_TYPE(key)->tp_name, Py_TYPE(value)->tp_name); + fprintf(stdout, "Key "); + PyObject_Print(key, stdout, Py_PRINT_RAW); + fprintf(stdout, " = "); + PyObject_Print(value, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + } + fprintf(stdout, "Initial reference counts first %zu last %zu\n", Py_REFCNT(self->first), Py_REFCNT(self->last)); +#endif + +// static char *kwlist[] = {"first", "last", "number", NULL}; +// +// PyArg_ParseTupleAndKeywords(args, state, "OOi", kwlist, &self->first, &self->last, &self->number); + +#if 0 + // PyObject *key = NULL; + Py_DECREF(self->first); + key = Py_BuildValue("s", "first"); + self->first = PyDict_GetItem(state, key); + Py_DECREF(key); + Py_INCREF(self->first); + + Py_DECREF(self->last); + key = Py_BuildValue("s", "last"); + self->last = PyDict_GetItem(state, key); + Py_DECREF(key); + Py_INCREF(self->last); + + key = Py_BuildValue("s", "number"); + self->number = PyLong_AsLong(PyDict_GetItem(state, key)); + Py_DECREF(key); +#endif + if (!PyDict_CheckExact(state)) { + PyErr_SetString(PyExc_ValueError, "Pickled object is not a dict."); + return NULL; + } + /* Version check. */ + /* Borrowed reference but no need to increment as we create a C long from it. */ + PyObject *temp = PyDict_GetItemString(state, PICKLE_VERSION_KEY); + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_Format(PyExc_KeyError, "No \"%s\" in pickled dict.", PICKLE_VERSION_KEY); + return NULL; + } + if (!PyLong_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict version key \"%s\" is not a long but type \"%s\".", + PICKLE_VERSION_KEY, + temp->ob_type->tp_name + ); + return NULL; + } + int pickle_version = (int) PyLong_AsLong(temp); + if (pickle_version != PICKLE_VERSION) { + PyErr_Format(PyExc_ValueError, "Pickle version mismatch. Got version %d but expected version %d.", + pickle_version, PICKLE_VERSION); + return NULL; + } + + temp = PyDict_GetItemString(state, "first"); /* Borrowed reference. */ + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"first\" in pickled dict."); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"first\" is not a str but type \"%s\".", + temp->ob_type->tp_name + ); + return NULL; + } + Py_DECREF(self->first); + self->first = temp; + Py_INCREF(self->first); + + temp = PyDict_GetItemString(state, "last"); /* Borrowed reference. */ + if (temp == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"last\" in pickled dict."); + return NULL; + } + if (!PyUnicode_Check(temp)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"last\" is not a str but type \"%s\".", + temp->ob_type->tp_name + ); + return NULL; + } + Py_DECREF(self->last); + self->last = temp; + Py_INCREF(self->last); + + /* Borrowed reference but no need to increment as we create a C long from it. */ + PyObject *number = PyDict_GetItemString(state, "number"); + if (number == NULL) { + /* PyDict_GetItemString does not set any error state so we have to. */ + PyErr_SetString(PyExc_KeyError, "No \"number\" in pickled dict."); + return NULL; + } + if (!PyLong_Check(number)) { + PyErr_Format( + PyExc_KeyError, + "Pickled dict key \"number\" is not an int but type \"%s\".", + number->ob_type->tp_name + ); + return NULL; + } + self->number = (int) PyLong_AsLong(number); +#if FPRINTF_DEBUG + fprintf(stdout, "Final reference counts first %zu last %zu\n", Py_REFCNT(self->first), + Py_REFCNT(self->last)); +#endif + Py_RETURN_NONE; +} + +static PyMethodDef Custom_methods[] = { + {"name", (PyCFunction) Custom_name, METH_NOARGS, + "Return the name, combining the first and last name" + }, + {"__getstate__", (PyCFunction) Custom___getstate__, METH_NOARGS, + "Return the state for pickling" + }, + {"__setstate__", (PyCFunction) Custom___setstate__, METH_O, + "Set the state from a pickle" + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "cPyExtPatt.cPickle.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = Custom_new, + .tp_init = (initproc) Custom_init, + .tp_dealloc = (destructor) Custom_dealloc, + .tp_members = Custom_members, + .tp_methods = Custom_methods, +}; + +static PyModuleDef cPicklemodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cPickle", + .m_doc = "Example module that creates a pickleable extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_cPickle(void) +{ + PyObject *m; + if (PyType_Ready(&CustomType) < 0) + return NULL; + + m = PyModule_Create(&cPicklemodule); + if (m == NULL) + return NULL; + + Py_INCREF(&CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + + return m; +} diff --git a/src/cpy/RefCount/cPyRefs.c b/src/cpy/RefCount/cPyRefs.c new file mode 100644 index 0000000..673b568 --- /dev/null +++ b/src/cpy/RefCount/cPyRefs.c @@ -0,0 +1,272 @@ +// +// PyReferences.c +// PythonExtensionPatterns +// +// Created by Paul Ross on 07/05/2014. +// Copyright (c) 2014 Paul Ross. All rights reserved. +// +#include "Python.h" + +//#include + +/* + * 'New', 'stolen' and 'borrowed' references. + * These terms are used throughout the Python documentation, they refer to + * who is the real owner of the reference i.e. whose job it is to finally + * dec_ref it (free it). + * + * This is all about programming by contract and each of reference types + * has a different contract. + */ + +/** ==== Manipulating reference counts directly ==== */ + +/** Returns the reference count of a PyObject. */ +static PyObject *ref_count(PyObject *Py_UNUSED(module), PyObject *pObj) { + assert(pObj); + return Py_BuildValue("n", pObj->ob_refcnt); +} + +/** Increments the reference count of a PyObject. + * Returns the original reference count. */ +static PyObject *inc_ref(PyObject *Py_UNUSED(module), PyObject *pObj) { + assert(pObj); + Py_ssize_t ret = pObj->ob_refcnt; + Py_INCREF(pObj); + return Py_BuildValue("n", ret); +} + +/** Decrements the reference count of a PyObject. + * Returns the original reference count. + * CAUTION: This may deallocate the object. + * */ +static PyObject *dec_ref(PyObject *Py_UNUSED(module), PyObject *pObj) { + assert(pObj); + Py_ssize_t ret = pObj->ob_refcnt; + Py_DECREF(pObj); + return Py_BuildValue("n", ret); +} + +/** ==== END: Manipulating reference counts directly ==== */ + + +/** ==== Example code for documentation. Not all of these are tested as they may segfault. ==== */ + +/** New reference. + * This is object creation and it is your job to dispose of it. + * + * The analogy with 'C' is the reference has been malloc'd and must be free'd + * by you. + */ +static PyObject *subtract_long(long a, long b) { + PyObject *pA, *pB, *r; + + pA = PyLong_FromLong(a); /* pA: New reference. */ + pB = PyLong_FromLong(b); /* pB: New reference. */ + r = PyNumber_Subtract(pA, pB); /* r: New reference. */ + Py_DECREF(pA); /* My responsibility to dec_ref. */ + Py_DECREF(pB); /* My responsibility to dec_ref. */ + return r; /* Callers responsibility to dec_ref. */ +} + +static PyObject *subtract_two_longs(PyObject *Py_UNUSED(module)) { + return subtract_long(421, 17); +} + +/** Create an object, dec-ref it and then try and print it. + * This may or may not work. + * Not pytest tested. + * */ +static PyObject *access_after_free(PyObject *Py_UNUSED(module)) { + PyObject *pA = PyLong_FromLong(1024L * 1024L); + fprintf( + stdout, + "%s(): Before Py_DECREF(0x%p) Ref count: %zd\n", + __FUNCTION__, (void *)pA, Py_REFCNT(pA) + ); + PyObject_Print(pA, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + + Py_DECREF(pA); + + fprintf( + stdout, + "%s(): After Py_DECREF(0x%p) Ref count: %zd\n", + __FUNCTION__, (void *)pA, Py_REFCNT(pA) + ); + PyObject_Print(pA, stdout, Py_PRINT_RAW); + fprintf(stdout, "\n"); + + Py_RETURN_NONE; +} + + +/** Stolen reference. + * This is object creation but where another object takes responsibility + * for dec_ref'ing (freeing) the object. + * These are quite rare; typical examples are object insertion into tuples + * lists, dicts etc. + * + * The analogy with C would be malloc'ing some memory, populating it and + * inserting that pointer into a linked list where the linked list promises + * to free the memory when that item in the list is removed. + */ +static PyObject *make_tuple(PyObject *Py_UNUSED(module)) { + PyObject *r; + PyObject *v; + + r = PyTuple_New(3); /* New reference. */ +// fprintf(stdout, "Ref count new: %zd\n", r->ob_refcnt); + v = PyLong_FromLong(1L); /* New reference. */ + /* PyTuple_SetItem steals the new reference v. */ + PyTuple_SetItem(r, 0, v); + /* This is fine. */ + v = PyLong_FromLong(2L); + PyTuple_SetItem(r, 1, v); + /* More common pattern. */ + PyTuple_SetItem(r, 2, PyUnicode_FromString("three")); + return r; /* Callers responsibility to dec_ref. */ +} + +/** Calls PySequence_DelItem() on each item. + * This decrements the reference count by one for each item. + */ +void delete_all_list_items(PyObject *pList) { + while (PyList_Size(pList) > 0) { + PySequence_DelItem(pList, PyList_Size(pList) - 1); + } +} + +/** 'Borrowed' reference this is when reading from an object, you get back a + * reference to something that the object still owns _and_ the container + * can dispose of at _any_ time. + * The problem is that you might want that reference for longer. + * + * Not pytest tested. + */ +static PyObject *pop_and_print_BAD(PyObject *Py_UNUSED(module), PyObject *pList) { + PyObject *pLast; + + pLast = PyList_GetItem(pList, PyList_Size(pList) - 1); + fprintf(stdout, "Ref count was: %zd\n", pLast->ob_refcnt); + /* ... stuff here ... */ + delete_all_list_items(pList); + /* ... more stuff here ... */ + fprintf(stdout, "Ref count now: %zd\n", pLast->ob_refcnt); + PyObject_Print(pLast, stdout, 0); /* Boom. */ + fprintf(stdout, "\n"); + Py_RETURN_NONE; +} + +/** The safer way, increment a borrowed reference. + * + * Not pytest tested. + * */ +static PyObject *pop_and_print_OK(PyObject *Py_UNUSED(module), PyObject *pList) { + PyObject *pLast; + + pLast = PyList_GetItem(pList, PyList_Size(pList) - 1); + fprintf(stdout, "Ref count was: %zd\n", pLast->ob_refcnt); + Py_INCREF(pLast); /* This is the crucial change: increment a borrowed reference. */ + fprintf(stdout, "Ref count now: %zd\n", pLast->ob_refcnt); + /* ... stuff here ... */ + delete_all_list_items(pList); + /* ... more stuff here ... */ + PyObject_Print(pLast, stdout, 0); + fprintf(stdout, "\n"); + Py_DECREF(pLast); + fprintf(stdout, "Ref count fin: %zd\n", pLast->ob_refcnt); + + Py_RETURN_NONE; +} + +/** + * This leaks new references by creating count number of longs of given value but + * never dec-refing them. + * + * Not pytest tested. + */ +static PyObject *leak_new_reference(PyObject *Py_UNUSED(module), + PyObject *args, PyObject *kwargs) { + PyObject *ret = NULL; + int value, count; + static char *kwlist[] = {"value", "count", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ii", kwlist, &value, + &count)) { + goto except; + } + fprintf(stdout, "loose_new_reference: value=%d count=%d\n", value, count); + for (int i = 0; i < count; ++i) { + PyLong_FromLong(value); /* New reference, leaked. */ + } + + Py_INCREF(Py_None); + ret = Py_None; + goto finally; +except: + Py_XDECREF(ret); + ret = NULL; +finally: + fprintf(stdout, "loose_new_reference: DONE\n"); + return ret; +} + +/** ==== END: Example code for documentation. Not all of these are tested as they may segfault. ==== */ + +static PyMethodDef cPyRefs_methods[] = { + { + "ref_count", (PyCFunction) ref_count, METH_O, + "Return the reference count a PyObject." + }, + { + "inc_ref", (PyCFunction) inc_ref, METH_O, + "Increment the reference count a PyObject. Returns the original reference count" + }, + { + "dec_ref", (PyCFunction) dec_ref, METH_O, + "Increment the reference count a PyObject. Returns the original reference count" + }, + { + "subtract_two_longs", (PyCFunction) subtract_two_longs, METH_NOARGS, + "Returns a new long by subtracting two longs in Python." + }, + { + "access_after_free", (PyCFunction) access_after_free, METH_NOARGS, + "Example of access after decrement reference." + }, + { + "make_tuple", (PyCFunction) make_tuple, METH_NOARGS, + "Creates a tuple by stealing new references." + }, + { + "pop_and_print_BAD", (PyCFunction) pop_and_print_BAD, METH_O, + "Borrowed refs, might segfault." + }, + { + "pop_and_print_OK", (PyCFunction) pop_and_print_OK, METH_O, + "Borrowed refs, should not segfault." + }, + { + "leak_new_reference", (PyCFunction) leak_new_reference, METH_VARARGS | METH_KEYWORDS, + "Leaks new references to longs." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cPyRefs_module = { + PyModuleDef_HEAD_INIT, + "cPyRefs", + "Examples of reference types in a 'C' extension.", + -1, + cPyRefs_methods, + NULL, /* inquiry m_reload */ + NULL, /* traverseproc m_traverse */ + NULL, /* inquiry m_clear */ + NULL, /* freefunc m_free */ +}; + +PyMODINIT_FUNC +PyInit_cPyRefs(void) { + return PyModule_Create(&cPyRefs_module); +} diff --git a/src/cpy/RefCount/cRefCount.c b/src/cpy/RefCount/cRefCount.c new file mode 100644 index 0000000..8a6cd4c --- /dev/null +++ b/src/cpy/RefCount/cRefCount.c @@ -0,0 +1,3123 @@ +// +// Created by Paul Ross on 20/10/2024. +// +// This explores reference counts with the Python C-API. +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +/* For access to new_unique_string().*/ +#include "pyextpatt_util.h" + +#define CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(return_value) \ +do { \ + if (PyErr_Occurred()) { \ + fprintf(stderr, "%s(): %s#%d entered with error.\n", \ + __FUNCTION__, __FILE_NAME__, __LINE__); \ + /* PyErr_Print(); */ \ + return return_value; \ + } \ +} while(0) + + +/** + * Decrement the reference counts of each set value by one. + * + * @param op The set. + * @return 0 on success, non-zero on failure in which case a Python Exception will have been set. + */ +static int +decref_set_values(PyObject *op) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(-1); + assert(!PyErr_Occurred()); + + if (!PySet_Check(op)) { + PyErr_Format(PyExc_ValueError, "Argument must be type set not type %s", Py_TYPE(op)->tp_name); + return 1; + } + /* https://docs.python.org/3/c-api/object.html#c.PyObject_GetIter + * This returns a new reference. */ + PyObject *iterator = PyObject_GetIter(op); + if (iterator == NULL) { + PyErr_Format(PyExc_ValueError, "Can not obtain iterator for type %s", Py_TYPE(op)->tp_name); + return 2; + } + PyObject *item; + /* https://docs.python.org/3/c-api/iter.html#c.PyIter_Next + * This returns a new reference. */ + while ((item = PyIter_Next(iterator))) { + Py_DECREF(item); /* This is the point of this function. */ + Py_DECREF(item); /* As this is a new reference. */ + } + Py_DECREF(iterator); + if (PyErr_Occurred()) { + return 3; + } + return 0; +} + +/** + * Checks the reference counts when creating and adding to a \c tuple. + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +tuple_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long result = 0; + PyObject *container = PyTuple_New(1); + if (container->ob_refcnt != 1) { + result |= 1 << 0; + } +// fprintf(stdout, "TRACE: tuple->ob_refcnt = %ld result %ld\n", tuple->ob_refcnt, result); + PyObject *value = PyLong_FromLong(123456); + if (value->ob_refcnt != 1) { + result |= 1 << 1; + } +// fprintf(stdout, "TRACE: value->ob_refcnt = %ld result %ld\n", value->ob_refcnt, result); + PyTuple_SET_ITEM(container, 0, value); + result |= value->ob_refcnt != 1; + if (value->ob_refcnt != 1) { + result |= 1 << 2; + } +// fprintf(stdout, "TRACE: value->ob_refcnt = %ld result %ld\n", value->ob_refcnt, result); + if (PyTuple_GET_ITEM(container, 0)->ob_refcnt != 1) { + result |= 1 << 3; + } +// fprintf(stdout, "TRACE: value->ob_refcnt = %ld result %ld\n", PyTuple_GET_ITEM(tuple, 0)->ob_refcnt, result); + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating a \c tuple with \c Py_BuildValue. + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +tuple_buildvalue_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + int result = 0; + PyObject *value_0 = PyLong_FromLong(123456); + if (value_0->ob_refcnt != 1) { + result |= 1 << 0; + } + PyObject *value_1 = PyLong_FromLong(1234567); + if (value_1->ob_refcnt != 1) { + result |= 1 << 1; + } + PyObject *container = Py_BuildValue("ii", value_0, value_1); + if (container->ob_type != &PyTuple_Type) { + result |= 1 << 2; + } + if (container->ob_refcnt != 1) { + result |= 1 << 3; + } + result |= value_0->ob_refcnt != 1; + if (value_0->ob_refcnt != 1) { + result |= 1 << 4; + } + result |= value_1->ob_refcnt != 1; + if (value_1->ob_refcnt != 1) { + result |= 1 << 5; + } + if (PyTuple_GET_ITEM(container, 0)->ob_refcnt != 1) { + result |= 1 << 6; + } + if (PyTuple_GET_ITEM(container, 1)->ob_refcnt != 1) { + result |= 1 << 7; + } + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating and adding to a \c list. + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +list_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long result = 0; + PyObject *container = PyList_New(1); + if (container->ob_refcnt != 1) { + result |= 1 << 0; + } + PyObject *value = PyLong_FromLong(123456); + if (value->ob_refcnt != 1) { + result |= 1 << 1; + } + PyList_SET_ITEM(container, 0, value); + if (value->ob_refcnt != 1) { + result |= 1 << 2; + } + if (PyList_GET_ITEM(container, 0)->ob_refcnt != 1) { + result |= 1 << 3; + } + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating a \c list with \c Py_BuildValue. + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +list_buildvalue_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + int result = 0; + PyObject *value_0 = PyLong_FromLong(123456); + if (value_0->ob_refcnt != 1) { + result |= 1 << 0; + } + PyObject *value_1 = PyLong_FromLong(1234567); + if (value_1->ob_refcnt != 1) { + result |= 1 << 1; + } + PyObject *container = Py_BuildValue("[ii]", value_0, value_1); + if (container->ob_type != &PyList_Type) { + result |= 1 << 2; + } + if (container->ob_refcnt != 1) { + result |= 1 << 3; + } + result |= value_0->ob_refcnt != 1; + if (value_0->ob_refcnt != 1) { + result |= 1 << 4; + } + result |= value_1->ob_refcnt != 1; + if (value_1->ob_refcnt != 1) { + result |= 1 << 5; + } + if (PyList_GET_ITEM(container, 0)->ob_refcnt != 1) { + result |= 1 << 6; + } + if (PyList_GET_ITEM(container, 1)->ob_refcnt != 1) { + result |= 1 << 7; + } + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating and adding to a \c set. + * + * The \c set object *does* increment the reference count. + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +set_no_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long result = 0; + PyObject *container = PySet_New(NULL); + if (container->ob_refcnt != 1) { + result |= 1 << 0; + } + PyObject *value = PyLong_FromLong(123456); + if (value->ob_refcnt != 1) { + result |= 1 << 1; + } + PySet_Add(container, value); + if (value->ob_refcnt != 2) { + result |= 1 << 2; + } + if (PySet_Size(container) != 1) { + result |= 1 << 3; + } + PyObject *pop = PySet_Pop(container); + if (pop->ob_refcnt != 2) { + result |= 1 << 4; + } + if (pop != value) { + result |= 1 << 5; + } + Py_DECREF(container); + if (value->ob_refcnt != 2) { + result |= 1 << 6; + } + Py_DECREF(value); + Py_DECREF(value); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating and adding to a \c set. + * This uses \c decref_set_values(). + * + * The \c set object *does* increment the reference count. + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +set_no_steals_decref(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long result = 0; + PyObject *container = PySet_New(NULL); + if (container->ob_refcnt != 1) { + result |= 1 << 0; + } + PyObject *value = PyLong_FromLong(123456); + if (value->ob_refcnt != 1) { + result |= 1 << 1; + } + PySet_Add(container, value); + if (value->ob_refcnt != 2) { + result |= 1 << 2; + } + if (PySet_Size(container) != 1) { + result |= 1 << 3; + } + // Use decref_set_values() + if (decref_set_values(container)) { + result |= 1 << 4; + } + if (value->ob_refcnt != 1) { + result |= 1 << 5; + } + PyObject *pop = PySet_Pop(container); + if (pop->ob_refcnt != 1) { + result |= 1 << 6; + } + if (PySet_Size(container) != 0) { + result |= 1 << 6; + } + Py_DECREF(container); + Py_DECREF(value); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating and adding to a \c dict. + * The \c dict object *does* increment the reference count for the key and the value. + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +dict_no_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long result = 0; + int result_shift = 0; + // Create the container + PyObject *container = PyDict_New(); + if (container->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Create the key and value. + PyObject *key = PyLong_FromLong(123456); + if (key->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + PyObject *value = PyLong_FromLong(1234567); + if (value->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Set the key and value. + PyDict_SetItem(container, key, value); + // Check the container size. + if (PyDict_Size(container) != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Check the key and value have incremented reference counts. + if (key->ob_refcnt != 2) { + result |= 1 << result_shift; + } + ++result_shift; + if (value->ob_refcnt != 2) { + result |= 1 << result_shift; + } + ++result_shift; + // Delete the key/value. + if (PyDict_DelItem(container, key)) { + result |= 1 << result_shift; + } + ++result_shift; + // Check the key and value have decremented reference counts. + if (key->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + if (value->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Clean up. + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating and adding to a \c dict. + * The \c dict object *does* increment the reference count for the key and the value. + * This demonstrates the canonical way of decrementing new objects immediately after calling PyDict_Set(). + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +dict_no_steals_decref_after_set(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long result = 0; + int result_shift = 0; + // Create the container + PyObject *container = PyDict_New(); + if (container->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Create the key and value. + PyObject *key = PyLong_FromLong(123456); + if (key->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + PyObject *value = PyLong_FromLong(1234567); + if (value->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Set the key and value. + PyDict_SetItem(container, key, value); + // Check the container size. + if (PyDict_Size(container) != 1) { + result |= 1 << result_shift; + } + ++result_shift; + if (key->ob_refcnt != 2) { + result |= 1 << result_shift; + } + ++result_shift; + if (value->ob_refcnt != 2) { + result |= 1 << result_shift; + } + ++result_shift; + // Now decrement the newly created objects + Py_DECREF(key); + // Check the key and value have single reference counts. + if (key->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + Py_DECREF(value); + if (value->ob_refcnt != 1) { + result |= 1 << result_shift; + } + ++result_shift; + // Delete the key/value. + if (PyDict_DelItem(container, key)) { + result |= 1 << result_shift; + } + ++result_shift; + if (PyDict_Size(container) != 0) { + result |= 1 << result_shift; + } + ++result_shift; + // Clean up. + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/** + * Checks the reference counts when creating a \c dict with \c Py_BuildValue. + * The \c dict object *does* increment the reference count for the key and the value. + * + * @param _unused_module + * @return Zero on success, non-zero on error. + */ +static PyObject * +dict_buildvalue_no_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + int result = 0; + int result_shift = 0; + PyObject *key = PyLong_FromLong(123456); + if (key->ob_refcnt != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + PyObject *value = PyLong_FromLong(1234567); + if (value->ob_refcnt != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Build the dict + PyObject *container = Py_BuildValue("{OO}", key, value); + if (container == NULL) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Check the container type. + if (container->ob_type != &PyDict_Type) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Check the container reference count. + if (container->ob_refcnt != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Check the container size. + if (PyDict_Size(container) != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() key->ob_refcnt=%ld value->ob_refcnt=%ld\n", key->ob_refcnt, value->ob_refcnt); + // Check the key and value have incremented reference counts. + if (key->ob_refcnt != 2) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + if (value->ob_refcnt != 2) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() key=%ld value=%ld\n", (long)key, (long)value); +// PyObject_Print(container, stdout, Py_PRINT_RAW); +// fprintf(stdout, "\n"); + + // Check the container has the key. +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() PyDict_Contains(container, key) %d\n", PyDict_Contains(container, key)); + if (PyDict_Contains(container, key) != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Delete the key/value. + if (PyDict_DelItem(container, key)) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Check the key and value have decremented reference counts. + if (key->ob_refcnt != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + if (value->ob_refcnt != 1) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } +// fprintf(stdout, "TRACE: dict_buildvalue_no_steals() result_shift=%d result %d\n", result_shift, result); + ++result_shift; + // Check the container size. + if (PyDict_Size(container) != 0) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } + ++result_shift; + // Check the container does not have the key. + if (PyDict_Contains(container, key) != 0) { + result |= 1 << result_shift; + return PyLong_FromLong(result); + } + ++result_shift; + // Clean up. + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(container); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +#define TEST_REF_COUNT_THEN_OR_RETURN_VALUE(variable, expected, commentary) \ + do { \ + Py_ssize_t _ref_count = Py_REFCNT(variable); \ + if (_ref_count != expected) { \ + fprintf( \ + stderr, \ + "Py_REFCNT(%s) != %ld but %ld. Test: %d Commentary: %s File: %s Line: %d\n", \ + #variable, expected, _ref_count, error_flag_position, commentary, __FILE__, __LINE__ \ + ); \ + return_value |= 1 << error_flag_position; \ + } \ + error_flag_position++; \ + } while (0) + + +#pragma mark - Teting Tuples + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SetItem. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * @param _unused_module + * @return 0 on success. + */ +static PyObject * +test_PyTuple_SetItem_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "PyTuple_New()"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value = new_unique_string(__FUNCTION__, NULL)"); + + if (PyTuple_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value after PyTuple_SetItem()"); + + get_item = PyTuple_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GetItem(container, 0); + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SET_ITEM. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * @param _unused_module + * @return 0 on success. + */ +static PyObject * +test_PyTuple_SET_ITEM_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + ref_count = Py_REFCNT(container); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyTuple_SET_ITEM(container, 0, value); + + ref_count = Py_REFCNT(value); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *get_item = PyTuple_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SetItem on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * This DOES leak an existing value contrary to the Python documentation. + * + * @param _unused_module + * @return None + */ +static PyObject * +test_PyTuple_SetItem_steals_replace(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + ref_count = Py_REFCNT(container); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + if (PyTuple_SetItem(container, 0, value_0)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != value_0) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now create a new value that will overwrite the old one. */ + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_1); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + /* Preserve the value_1 so that we can see Py_DECREF(container) decrements it. */ + Py_INCREF(value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 2); + + /* This will overwrite value_0 leaving it with a reference count of 1.*/ + if (PyTuple_SetItem(container, 0, value_1)) { + fprintf(stdout, "PyTuple_SetItem(container, 0, value_1)\n"); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Previous value is decremented. */ + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + fprintf(stdout, "Py_REFCNT(value_0) != 1 but %ld\n", Py_REFCNT(value_0)); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + ref_count = Py_REFCNT(value_1); + if (ref_count != 2) { + fprintf(stdout, "Py_REFCNT(value_1) != 2 but %ld\n", Py_REFCNT(value_1)); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != value_1) { + fprintf(stdout, "get_item != value_1\n"); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + ref_count = Py_REFCNT(get_item); + if (ref_count != 2) { + fprintf(stdout, "Py_REFCNT(get_item) != 1 but %ld\n", Py_REFCNT(get_item)); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + ref_count = Py_REFCNT(value_1); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(value_1); + + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * A function that checks whether a tuple steals a reference when using PyTuple_SET_ITEM on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * @param _unused_module + * @return None + */ +static PyObject * +test_PyTuple_SET_ITEM_steals_replace(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 1L, "Create value_0."); + + PyTuple_SET_ITEM(container, 0, value_0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 1L, "PyTuple_SET_ITEM(container, 0, value_0);"); + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != value_0) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "PyTuple_GET_ITEM(container, 0);"); + + /* Now create a new value that will overwrite the old one. */ + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 1L, "Create value_1"); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 2L, "Py_INCREF(value_0);"); + + /* Preserve the value_1 so that we can see Py_DECREF(container) decrements it. */ + Py_INCREF(value_1); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 2L, "Py_INCREF(value_1);"); + + /* This will overwrite value_0 but not dec ref value_0 leaving + * value_0 still with a reference count of 2. + * This is a leak. */ + PyTuple_SET_ITEM(container, 0, value_1); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 2L, + "Py_REFCNT(value_0) after PyTuple_SET_ITEM(container, 0, value_1);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 2L, + "Py_REFCNT(value_1) after PyTuple_SET_ITEM(container, 0, value_1);"); + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != value_1) { + fprintf(stdout, "get_item != value_1\n"); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "PyTuple_GET_ITEM(container, 0);"); + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 1L, "value_1 after Py_DECREF(container);"); + + Py_DECREF(value_1); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 2L, "value_0 after Py_DECREF(container);"); + Py_DECREF(value_0); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * Function that check the behaviour of PyTuple_SetItem() when setting the *same* value. + * See also dbg_PyTuple_SetItem_replace_with_same() in src/cpy/Containers/DebugContainers.c + * + * @param _unused_module + * @return + */ +static PyObject * +test_PyTuple_SetItem_replace_same(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyTuple_New(1);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + /* Set the first time. */ + if (PyTuple_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After first PyTuple_SetItem(container, 0, value);"); + /*Get and test the first item. */ + get_item = PyTuple_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyTuple_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now incref the value so we can prevent a SIGSEGV with a double PyTuple_SetItem(). */ + Py_INCREF(value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After Py_INCREF(value);"); + + /* Second PyTuple_SetItem(). */ + /* This will overwrite value leaving it with a reference count of 1 if it wasn't for the Py_INCREF(value); above.*/ + if (PyTuple_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + /* This checks that PyTuple_SetItem() has decremented the original reference count. */ + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After second PyTuple_SetItem(container, 0, value);"); + + /* Check the value is the same. */ + get_item = PyTuple_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyTuple_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Decref the container. value will be ree'd. Double check values reference count is 1. */ + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "Before Py_DECREF(container);"); + Py_DECREF(container); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_SET_ITEM_replace_same(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyTuple_New(1);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + /* Set the first time. Does not alter reference count. */ + PyTuple_SET_ITEM(container, 0, value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After first PyTuple_SET_ITEM(container, 0, value);"); + /* Get and test the first item. */ + get_item = PyTuple_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyTuple_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Second PyTuple_SET_ITEM(). Does not alter reference count. */ + PyTuple_SET_ITEM(container, 0, value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After second PyTuple_SET_ITEM(container, 0, value);"); + + /* Check the value is the same. */ + get_item = PyTuple_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyTuple_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_INCREF(value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "Before Py_DECREF(container);"); + /* Decref the container. value will be decref'd. Double check values reference count is 1. */ + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_SetItem_NULL(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + PyTuple_SetItem(container, 0, NULL); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_SET_ITEM_NULL(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + ref_count = Py_REFCNT(container); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyTuple_SET_ITEM(container, 0, NULL); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_SetIem_NULL_SetItem(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + if (PyTuple_SetItem(container, 0, NULL)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now set a non-null value. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "Create value."); + + /* Increment so we can check after deleting the container. */ + Py_INCREF(value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "Py_INCREF(value);"); + + /* Set, replacing NULL. */ + if (PyTuple_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item == NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "Py_INCREF(value);"); + + Py_DECREF(value); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_SET_ITEM_NULL_SET_ITEM(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyTuple_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + PyTuple_SET_ITEM(container, 0, NULL); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now set a non-null value. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "PyObject *value = new_unique_string(__FUNCTION__, NULL);."); + + /* Increment so we can check after deleting the container. */ + Py_INCREF(value); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "PyObject *value after Py_INCREF."); + + /* Set, replacing NULL. */ + PyTuple_SET_ITEM(container, 0, value); + + get_item = PyTuple_GET_ITEM(container, 0); + if (get_item == NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "PyObject *value after Py_DECREF(container);."); + + Py_DECREF(value); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_SetItem_fails_not_a_tuple(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyList_New(1); + if (!container) { + return NULL; + } + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* This should fail. */ + if (PyTuple_SetItem(container, 0, value)) { + /* DO NOT do this, it has been done by the failure PyTuple_SetItem(). */ + /* Py_DECREF(value); */ + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; + } + Py_DECREF(value); + Py_DECREF(container); + PyErr_Format(PyExc_RuntimeError, "Should have raised an error."); + return NULL; +} + +static PyObject * +test_PyTuple_SetItem_fails_out_of_range(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyTuple_New(1); + if (!container) { + return NULL; + } + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* This should fail. */ + if (PyTuple_SetItem(container, 1, value)) { + /* DO NOT do this, it has been done by the failure PyTuple_SetItem(). */ + /* Py_DECREF(value); */ + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; + } + Py_DECREF(value); + Py_DECREF(container); + PyErr_Format(PyExc_RuntimeError, "Should have raised an error."); + return NULL; +} + +static PyObject * +test_PyTuple_Py_PyTuple_Pack(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + long return_value = 0L; + int error_flag_position = 0; + + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_a, 1L, + "After PyObject *value_a = new_unique_string(__FUNCTION__, NULL);"); + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 1L, + "After PyObject *value_b = new_unique_string(__FUNCTION__, NULL);"); + + PyObject *container = PyTuple_Pack(2, value_a, value_b); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, + "After PyObject *container = PyTuple_Pack(2, value_a, value_b);"); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_a, 2L, + "value_a after PyObject *container = PyTuple_Pack(2, value_a, value_b);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 2L, + "value_b after PyObject *container = PyTuple_Pack(2, value_a, value_b);"); + + Py_DECREF(container); + + /* Leaks: */ + assert(Py_REFCNT(value_a) == 1); + assert(Py_REFCNT(value_b) == 1); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_a, 1L, "value_a after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 1L, "value_b after Py_DECREF(container);"); + /* Fix leaks: */ + Py_DECREF(value_a); + Py_DECREF(value_b); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyTuple_Py_BuildValue(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; +// PyObject *get_item = NULL; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + + PyObject *container = Py_BuildValue("(O)", value); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After PyObject *container = Py_BuildValue(\"(O)\", value);"); + + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Container"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *container = Py_BuildValue(\"(O)\", value);"); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +#pragma mark - Testing Lists + +/** + * A function that checks whether a tuple steals a reference when using PyList_SetItem. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * @param _unused_module + * @return 0 on success. + */ +static PyObject * +test_PyList_SetItem_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "PyList_New()"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value = new_unique_string(__FUNCTION__, NULL)"); + + if (PyList_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value after PyList_SetItem()"); + + get_item = PyList_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GetItem(container, 0); + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * A function that checks whether a tuple steals a reference when using PyList_SET_ITEM. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * @param _unused_module + * @return 0 on success. + */ +static PyObject * +test_PyList_SET_ITEM_steals(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + ref_count = Py_REFCNT(container); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyList_SET_ITEM(container, 0, value); + + ref_count = Py_REFCNT(value); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *get_item = PyList_GET_ITEM(container, 0); + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + /* NO as container deals with this. */ + /* Py_DECREF(value); */ + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * A function that checks whether a tuple steals a reference when using PyList_SetItem on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * This DOES leak an existing value contrary to the Python documentation. + * + * @param _unused_module + * @return None + */ +static PyObject * +test_PyList_SetItem_steals_replace(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + ref_count = Py_REFCNT(container); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + if (PyList_SetItem(container, 0, value_0)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != value_0) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + ref_count = Py_REFCNT(get_item); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now create a new value that will overwrite the old one. */ + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(value_1); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + ref_count = Py_REFCNT(value_0); + assert(ref_count == 2); + + /* Preserve the value_1 so that we can see Py_DECREF(container) decrements it. */ + Py_INCREF(value_1); + ref_count = Py_REFCNT(value_1); + assert(ref_count == 2); + + /* This will overwrite value_0 leaving it with a reference count of 1.*/ + if (PyList_SetItem(container, 0, value_1)) { + fprintf(stdout, "PyList_SetItem(container, 0, value_1)\n"); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Previous value is decremented. */ + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + fprintf(stdout, "Py_REFCNT(value_0) != 1 but %ld\n", Py_REFCNT(value_0)); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + ref_count = Py_REFCNT(value_1); + if (ref_count != 2) { + fprintf(stdout, "Py_REFCNT(value_1) != 2 but %ld\n", Py_REFCNT(value_1)); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != value_1) { + fprintf(stdout, "get_item != value_1\n"); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + ref_count = Py_REFCNT(get_item); + if (ref_count != 2) { + fprintf(stdout, "Py_REFCNT(get_item) != 1 but %ld\n", Py_REFCNT(get_item)); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + ref_count = Py_REFCNT(value_1); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(value_1); + + ref_count = Py_REFCNT(value_0); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +/** + * A function that checks whether a tuple steals a reference when using PyList_SET_ITEM on an existing item. + * This can be stepped through in the debugger. + * asserts are use for the test so this is expected to be run in DEBUG mode. + * + * @param _unused_module + * @return None + */ +static PyObject * +test_PyList_SET_ITEM_steals_replace(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + PyObject *value_0 = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 1L, "Create value_0."); + + PyList_SET_ITEM(container, 0, value_0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 1L, "PyList_SET_ITEM(container, 0, value_0);"); + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != value_0) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "PyList_GET_ITEM(container, 0);"); + + /* Now create a new value that will overwrite the old one. */ + PyObject *value_1 = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 1L, "Create value_1"); + + /* Preserve the value_0 as this reference count is about to be decremented. */ + Py_INCREF(value_0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 2L, "Py_INCREF(value_0);"); + + /* Preserve the value_1 so that we can see Py_DECREF(container) decrements it. */ + Py_INCREF(value_1); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 2L, "Py_INCREF(value_1);"); + + /* This will overwrite value_0 but not dec ref value_0 leaving + * value_0 still with a reference count of 2. + * This is a leak. */ + PyList_SET_ITEM(container, 0, value_1); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 2L, + "Py_REFCNT(value_0) after PyList_SET_ITEM(container, 0, value_1);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 2L, + "Py_REFCNT(value_1) after PyList_SET_ITEM(container, 0, value_1);"); + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != value_1) { + fprintf(stdout, "get_item != value_1\n"); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "PyList_GET_ITEM(container, 0);"); + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_1, 1L, "value_1 after Py_DECREF(container);"); + + Py_DECREF(value_1); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_0, 2L, "value_0 after Py_DECREF(container);"); + Py_DECREF(value_0); + Py_DECREF(value_0); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SetItem_replace_same(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyList_New(1);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + /* Set the first time. */ + if (PyList_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After first PyList_SetItem(container, 0, value);"); + /*Get and test the first item. */ + get_item = PyList_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyList_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now incref the value so we can prevent a SIGSEGV with a double PyList_SetItem(). */ + Py_INCREF(value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After Py_INCREF(value);"); + + /* Second PyList_SetItem(). */ + /* This will overwrite value leaving it with a reference count of 1 if it wasn't for the Py_INCREF(value); above.*/ + if (PyList_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + /* This checks that PyList_SetItem() has decremented the original reference count. */ + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After second PyList_SetItem(container, 0, value);"); + + /* Check the value is the same. */ + get_item = PyList_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyList_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Decref the container. value will be ree'd. Double check values reference count is 1. */ + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "Before Py_DECREF(container);"); + Py_DECREF(container); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SET_ITEM_replace_same(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyList_New(1);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + /* Set the first time. Does not alter reference count. */ + PyList_SET_ITEM(container, 0, value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After first PyList_SET_ITEM(container, 0, value);"); + /* Get and test the first item. */ + get_item = PyList_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyList_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Second PyList_SET_ITEM(). Does not alter reference count. */ + PyList_SET_ITEM(container, 0, value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After second PyList_SET_ITEM(container, 0, value);"); + + /* Check the value is the same. */ + get_item = PyList_GET_ITEM(container, 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "After PyList_GET_ITEM(container, 0);"); + if (get_item != value) { + fprintf(stderr, "get_item != value at File: %s Line: %d\n", __FILE__, __LINE__); + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_INCREF(value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "Before Py_DECREF(container);"); + /* Decref the container. value will be decref'd. Double check values reference count is 1. */ + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SetItem_NULL(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + PyList_SetItem(container, 0, NULL); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SET_ITEM_NULL(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + Py_ssize_t ref_count; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + ref_count = Py_REFCNT(container); + if (ref_count != 1) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyList_SET_ITEM(container, 0, NULL); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SetIem_NULL_SetItem(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + if (PyList_SetItem(container, 0, NULL)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now set a non-null value. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "Create value."); + + /* Increment so we can check after deleting the container. */ + Py_INCREF(value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "Py_INCREF(value);"); + + /* Set, replacing NULL. */ + if (PyList_SetItem(container, 0, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item == NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "Py_INCREF(value);"); + + Py_DECREF(value); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SET_ITEM_NULL_SET_ITEM(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyList_New(1); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Create container."); + + PyList_SET_ITEM(container, 0, NULL); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + get_item = PyList_GET_ITEM(container, 0); + if (get_item != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + /* Now set a non-null value. */ + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "PyObject *value = new_unique_string(__FUNCTION__, NULL);."); + + /* Increment so we can check after deleting the container. */ + Py_INCREF(value); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "PyObject *value after Py_INCREF."); + + /* Set, replacing NULL. */ + PyList_SET_ITEM(container, 0, value); + + get_item = PyList_GET_ITEM(container, 0); + if (get_item == NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "PyObject *value after Py_DECREF(container);."); + + Py_DECREF(value); + + if (PyErr_Occurred()) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + assert(!PyErr_Occurred()); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_SetItem_fails_not_a_list(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyTuple_New(1); + if (!container) { + return NULL; + } + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* This should fail. */ + if (PyList_SetItem(container, 0, value)) { + /* DO NOT do this, it has been done by the failure PyTuple_SetItem(). */ + /* Py_DECREF(value); */ + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; + } + Py_DECREF(value); + Py_DECREF(container); + PyErr_Format(PyExc_RuntimeError, "Should have raised an error."); + return NULL; +} + +static PyObject * +test_PyList_SetItem_fails_out_of_range(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyList_New(1); + if (!container) { + return NULL; + } + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* This should fail. */ + if (PyList_SetItem(container, 1, value)) { + /* DO NOT do this, it has been done by the failure PyTuple_SetItem(). */ + /* Py_DECREF(value); */ + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; + } + Py_DECREF(value); + Py_DECREF(container); + PyErr_Format(PyExc_RuntimeError, "Should have raised an error."); + return NULL; +} + +static PyObject * +test_PyList_Append(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PyList_New(0); + if (!container) { + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyList_New(0);"); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + + if (PyList_Append(container, value)) { + assert(PyErr_Occurred()); + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After PyList_Append(container, value);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyList_Append(container, value);"); + + Py_DECREF(value); + Py_DECREF(container); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_Append_fails_not_a_list(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyTuple_New(1); + if (!container) { + return NULL; + } + PyObject *value = new_unique_string(__FUNCTION__, NULL); + + int result = PyList_Append(container, value); + assert(result); + Py_DECREF(value); + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; +} + +static PyObject * +test_PyList_Append_fails_NULL(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyList_New(0); + if (!container) { + return NULL; + } + + int result = PyList_Append(container, NULL); + assert(result); + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; +} + +static PyObject * +test_PyList_Insert(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PyList_New(0); + if (!container) { + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyList_New(0);"); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + + if (PyList_Insert(container, 0L, value)) { + assert(PyErr_Occurred()); + return NULL; + } + if (PyList_GET_SIZE(container) != 1) { + Py_DECREF(container); + Py_DECREF(value); + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After PyList_Append(container, value);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyList_Append(container, value);"); + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After Py_DECREF(container);"); + Py_DECREF(value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_Insert_Is_Truncated(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PyList_New(0); + if (!container) { + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyList_New(0);"); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + + if (PyList_Insert(container, 4L, value)) { + assert(PyErr_Occurred()); + return NULL; + } + if (PyList_GET_SIZE(container) != 1) { + Py_DECREF(container); + Py_DECREF(value); + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After PyList_Append(container, value);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyList_Append(container, value);"); + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After Py_DECREF(container);"); + Py_DECREF(value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_Insert_Negative_Index(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PyList_New(0); + if (!container) { + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyObject *container = PyList_New(0);"); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + + if (PyList_Insert(container, -1L, value)) { + assert(PyErr_Occurred()); + return NULL; + } + if (PyList_GET_SIZE(container) != 1) { + Py_DECREF(container); + Py_DECREF(value); + return NULL; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After PyList_Append(container, value);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "After PyList_Append(container, value);"); + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After Py_DECREF(container);"); + Py_DECREF(value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyList_Insert_fails_not_a_list(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyTuple_New(1); + if (!container) { + return NULL; + } + PyObject *value = new_unique_string(__FUNCTION__, NULL); + + int result = PyList_Insert(container, 1L, value); + assert(result); + Py_DECREF(value); + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; +} + +static PyObject * +test_PyList_Insert_fails_NULL(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyList_New(0); + if (!container) { + return NULL; + } + + int result = PyList_Insert(container, 1L, NULL); + assert(result); + Py_DECREF(container); + assert(PyErr_Occurred()); + return NULL; +} + +static PyObject * +test_PyList_Py_BuildValue(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; +// PyObject *get_item = NULL; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *value = new_unique_string(__FUNCTION__, NULL);"); + + PyObject *container = Py_BuildValue("[O]", value); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "After PyObject *container = Py_BuildValue(\"(O)\", value);"); + + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "Container"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "After PyObject *container = Py_BuildValue(\"(O)\", value);"); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +#pragma mark - Testing Dictionaries + +static PyObject * +test_PyDict_SetItem_increments(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyDict_New(); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "PyDict_New()"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key = new_unique_string(__FUNCTION__, NULL)"); + PyObject *value_a = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_a, 1L, "value = new_unique_string(__FUNCTION__, NULL)"); + + if (PyDict_SetItem(container, key, value_a)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_a, 2L, "value_a after PyDict_SetItem()"); + + get_item = PyDict_GetItem(container, key); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "get_item = PyDict_GetItem(container, key);"); + if (get_item != value_a) { + fprintf(stderr, "get_item = PyDict_GetItem(container, key); is not value_a\n"); + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + /* Now replace value_a with a new value, value_b. */ + PyObject *value_b = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 1L, "value_a = new_unique_string(__FUNCTION__, NULL)"); + + if (PyDict_SetItem(container, key, value_b)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_a, 1L, "value_a after PyList_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 2L, "value_b after PyList_SetItem()"); + + get_item = PyDict_GetItem(container, key); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "get_item = PyDict_GetItem(container, key);"); + if (get_item != value_b) { + fprintf(stderr, "get_item = PyDict_GetItem(container, key); is not value_b\n"); + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + // Replace with existing key/value_b. Reference counts should remain the same. + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key before PyDict_SetItem(container, key, value_b)"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 2L, "value_b before PyDict_SetItem(container, key, value_b)"); + if (PyDict_SetItem(container, key, value_b)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key before PyDict_SetItem(container, key, value_b)"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 2L, "value_b before PyDict_SetItem(container, key, value_b)"); + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_b, 1L, "value_b after Py_DECREF(container);"); + Py_DECREF(key); + Py_DECREF(value_a); + Py_DECREF(value_b); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyDict_SetItem_fails_not_a_dict(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyList_New(0); + if (!container) { + assert(PyErr_Occurred()); + return NULL; + } + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* This should fail. */ + if (PyDict_SetItem(container, key, value)) { + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(value); + assert(PyErr_Occurred()); + return NULL; + } + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(value); + PyErr_Format(PyExc_RuntimeError, "Should have raised an error."); + return NULL; +} + +static PyObject * +test_PyDict_SetItem_fails_not_hashable(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + + PyObject *container = PyDict_New(); + if (!container) { + assert(PyErr_Occurred()); + return NULL; + } + PyObject *key = PyList_New(0);; + PyObject *value = new_unique_string(__FUNCTION__, NULL); + /* This should fail. */ + if (PyDict_SetItem(container, key, value)) { + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(value); + assert(PyErr_Occurred()); + return NULL; + } + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(value); + PyErr_Format(PyExc_RuntimeError, "Should have raised an error."); + return NULL; +} + +static PyObject * +test_PyDict_SetDefault_default_unused(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "New key"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "New value"); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + goto finally; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PyDict_SetItem()"); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "value after PyDict_GetItem()"); + + /* Now check PyDict_SetDefault() which does not use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "New value_default"); + + get_item = PyDict_SetDefault(container, key, value_default); + assert(get_item == value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetDefault()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PyDict_SetDefault()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "value after PyDict_SetDefault()"); + + Py_DECREF(container); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value_default); + + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +PyObject * +test_PyDict_SetDefault_default_used(void) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "container after PyObject *container = PyDict_New();"); + + /* Do not do this so the default is invoked. + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + */ + + /* Now check PyDict_SetDefault() which *does* use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "container after PyObject *container = PyDict_New();"); + + get_item = PyDict_SetDefault(container, key, value_default); + if (!get_item) { + assert(0); + } + assert(PyDict_Size(container) == 1); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetDefault()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 2L, "value_default after PyDict_SetDefault()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "get_item after PyDict_SetDefault()"); + assert(get_item == value_default); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "value_default after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 1L, "get_item after Py_DECREF(container);"); + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value_default); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +static PyObject * +test_PyDict_SetDefaultRef_default_unused(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "New key"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "New value"); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + goto finally; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PyDict_SetItem()"); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "get_item after PyDict_GetItem()"); + + /* Now check PyDict_SetDefault() which does not use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "New value_default"); + + PyObject *result = NULL; + int ret_val = PyDict_SetDefaultRef(container, key, value_default, &result); + if (ret_val != 1) { + return_value = -1; + } + assert(result == value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetDefaultRef()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 3L, "value after PyDict_SetDefaultRef()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(result, 3L, "value after PyDict_SetDefaultRef()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "value_default after PyDict_SetDefaultRef()"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(result, 2L, "value after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "value_default after Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value); + Py_DECREF(value_default); + + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +PyObject * +test_PyDict_SetDefaultRef_default_used(void) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PyDict_New(); + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "container after PyObject *container = PyDict_New();"); + + /* Do not do this so the default is invoked. + if (PyDict_SetItem(container, key, value)) { + assert(0); + } + */ + + /* Now check PyDict_SetDefault() which *does* use the default. */ + PyObject *value_default = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *result = NULL; + int ret_val = PyDict_SetDefaultRef(container, key, value_default, &result); + if (ret_val != 0) { + return_value = -1; + } + assert(result == value_default); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetDefaultRef()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 3L, "value after PyDict_SetDefaultRef()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(result, 3L, "result after PyDict_SetDefaultRef()"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value_default, 2L, "value after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(result, 2L, "result after Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value_default); + Py_DECREF(value_default); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + + +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +static PyObject * +test_PyDict_GetItem(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item = NULL; + + PyObject *container = PyDict_New(); + if (!container) { + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "PyDict_New()"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key = new_unique_string(__FUNCTION__, NULL)"); + if (PyDict_GetItem(container, key) != NULL) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value = new_unique_string(__FUNCTION__, NULL)"); + + if (PyDict_SetItem(container, key, value)) { + return_value |= 1 << error_flag_position; + } + error_flag_position++; + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value_a after PyDict_SetItem()"); + + get_item = PyDict_GetItem(container, key); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "get_item = PyDict_GetItem(container, key);"); + if (get_item != value) { + fprintf(stderr, "get_item = PyDict_GetItem(container, key); is not value_a\n"); + return_value |= 1 << error_flag_position; + goto finally; + } + error_flag_position++; + + Py_DECREF(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value_b after Py_DECREF(container);"); + Py_DECREF(key); + Py_DECREF(value); + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +static PyObject * +test_PyDict_Pop_key_present(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + PyObject *get_item; + + PyObject *container = PyDict_New(); + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "New key"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "New value"); + + if (PyDict_SetItem(container, key, value)) { + assert(0); + goto finally; + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 2L, "key after PyDict_SetItem()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PyDict_SetItem()"); + + get_item = PyDict_GetItem(container, key); + assert(get_item == value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(get_item, 2L, "get_item after PyDict_GetItem()"); + + PyObject *result = NULL; + int ret_val = PyDict_Pop(container, key, &result); + if (ret_val != 1) { + return PyLong_FromLong(-1); + } + assert(result == value); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after PyDict_Pop()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PyDict_Pop()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(result, 2L, "result after PyDict_Pop()"); + + Py_DECREF(container); + + /* Duplicate of above as Py_DECREF(container); does not affect the key/value. */ + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(result, 2L, "result after Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(value); + + finally: + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PyDict_Pop_key_absent(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PyDict_New(); + assert(container); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PyDict_New();"); + + PyObject *key = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "New key"); + + /* Not inserted into the dict, just used so that result references it. */ + PyObject *dummy_value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(dummy_value, 1L, "New value"); + + PyObject *result = dummy_value; + int ret_val = PyDict_Pop(container, key, &result); + if (ret_val != 0) { + return PyLong_FromLong(-1); + } + assert(result == NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after PyDict_Pop()"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(dummy_value, 1L, "value after PyDict_Pop()"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(key, 1L, "key after Py_DECREF(container);"); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(dummy_value, 1L, "value after PyDict_Pop()"); + + /* Clean up. */ + Py_DECREF(key); + Py_DECREF(dummy_value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +static PyObject * +test_PySet_Add(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PySet_New(NULL); + assert(container); + assert(PySet_GET_SIZE(container) == 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PySet_New(NULL);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "New value"); + + int ret_val = PySet_Add(container, value); + if (ret_val != 0) { + return PyLong_FromLong(-1); + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PySet_Add()"); + + // Add duplicate. + ret_val = PySet_Add(container, value); + if (ret_val != 0) { + return PyLong_FromLong(-1); + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after second PySet_Add()"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value after Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PySet_Discard(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PySet_New(NULL); + assert(container); + assert(PySet_GET_SIZE(container) == 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PySet_New(NULL);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "New value"); + + int ret_val = PySet_Add(container, value); + if (ret_val != 0) { + return PyLong_FromLong(-1); + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PySet_Add()"); + + // Discard. + if (PySet_Discard(container, value) != 1) { + return PyLong_FromLong(-2); + } + assert(PySet_GET_SIZE(container) == 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value after PySet_Discard(container, value)"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "value after Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +static PyObject * +test_PySet_Pop(PyObject *Py_UNUSED(module)) { + CHECK_FOR_PYERROR_ON_FUNCTION_ENTRY(NULL); + assert(!PyErr_Occurred()); + long return_value = 0L; + int error_flag_position = 0; + + PyObject *container = PySet_New(NULL); + assert(container); + assert(PySet_GET_SIZE(container) == 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(container, 1L, "container after PyObject *container = PySet_New(NULL);"); + + PyObject *value = new_unique_string(__FUNCTION__, NULL); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 1L, "New value"); + + int ret_val = PySet_Add(container, value); + if (ret_val != 0) { + return PyLong_FromLong(-1); + } + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PySet_Add()"); + + // Pop. + PyObject *popped_value = PySet_Pop(container); + assert(popped_value == value); + assert(PySet_GET_SIZE(container) == 0); + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after PySet_Pop(container)"); + + Py_DECREF(container); + + TEST_REF_COUNT_THEN_OR_RETURN_VALUE(value, 2L, "value after Py_DECREF(container);"); + + /* Clean up. */ + Py_DECREF(value); + Py_DECREF(value); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(return_value); +} + +#define MODULE_NOARGS_ENTRY(name, doc) \ + { \ + #name, \ + (PyCFunction) name, \ + METH_NOARGS, \ + doc, \ + } + +static PyMethodDef module_methods[] = { + MODULE_NOARGS_ENTRY(tuple_steals, "Checks that PyTuple_SET_ITEM steals a reference count."), + MODULE_NOARGS_ENTRY(tuple_buildvalue_steals, "Checks that Py_BuildValue tuple steals a reference count."), + MODULE_NOARGS_ENTRY(list_steals, "Checks that PyTuple_SET_ITEM list steals a reference count."), + MODULE_NOARGS_ENTRY(list_buildvalue_steals, "Checks that Py_BuildValue list steals a reference count."), + MODULE_NOARGS_ENTRY(set_no_steals, "Checks that a set increments a reference count."), + MODULE_NOARGS_ENTRY(set_no_steals_decref, + "Checks that a set increments a reference count and uses decref_set_values."), + MODULE_NOARGS_ENTRY(dict_no_steals, "Checks that a dict increments a reference counts for key and value."), + MODULE_NOARGS_ENTRY(dict_no_steals_decref_after_set, + "Checks that a dict increments a reference counts for key and value." + " They are decremented after PyDict_Set()" + ), + MODULE_NOARGS_ENTRY(dict_buildvalue_no_steals, + "Checks that a Py_BuildValue dict increments a reference counts for key and value."), +#pragma mark - Testing Tuples + /* Test ref counts with container APIs. */ + MODULE_NOARGS_ENTRY(test_PyTuple_SetItem_steals, "Check that PyTuple_SetItem() steals a reference."), + MODULE_NOARGS_ENTRY(test_PyTuple_SET_ITEM_steals, "Check that PyTuple_SET_ITEM() steals a reference."), + MODULE_NOARGS_ENTRY(test_PyTuple_SetItem_steals_replace, + "Check that PyTuple_SetItem() steals a reference on replacement."), + MODULE_NOARGS_ENTRY(test_PyTuple_SET_ITEM_steals_replace, + "Check that PyTuple_SET_ITEM() steals a reference on replacement."), + MODULE_NOARGS_ENTRY(test_PyTuple_SetItem_replace_same, + "Check how PyTuple_SetItem() behaves on replacement of the same value."), + MODULE_NOARGS_ENTRY(test_PyTuple_SET_ITEM_replace_same, + "Check how PyTuple_SET_ITEM() behaves on replacement of the same value."), + MODULE_NOARGS_ENTRY(test_PyTuple_SetItem_NULL, "Check that PyTuple_SetItem() with NULL does not error."), + MODULE_NOARGS_ENTRY(test_PyTuple_SET_ITEM_NULL, "Check that PyTuple_SET_ITEM() with NULL does not error."), + MODULE_NOARGS_ENTRY(test_PyTuple_SetIem_NULL_SetItem, + "Check that PyTuple_SetItem() with NULL then with an object does not error."), + MODULE_NOARGS_ENTRY(test_PyTuple_SET_ITEM_NULL_SET_ITEM, + "Check that PyTuple_SET_ITEM() with NULL then with an object does not error."), + MODULE_NOARGS_ENTRY(test_PyTuple_SetItem_fails_not_a_tuple, + "Check that PyTuple_SET_ITEM() fails when not a tuple."), + MODULE_NOARGS_ENTRY(test_PyTuple_SetItem_fails_out_of_range, + "Check that PyTuple_SET_ITEM() fails when index out of range."), + MODULE_NOARGS_ENTRY(test_PyTuple_Py_PyTuple_Pack, + "Check that Py_PyTuple_Pack() increments reference counts."), + MODULE_NOARGS_ENTRY(test_PyTuple_Py_BuildValue, + "Check that Py_BuildValue() increments reference counts."), +#pragma mark - Testing Lists + /* Test ref counts with container APIs. */ + MODULE_NOARGS_ENTRY(test_PyList_SetItem_steals, "Check that PyList_SetItem() steals a reference."), + MODULE_NOARGS_ENTRY(test_PyList_SET_ITEM_steals, "Check that PyList_SET_ITEM() steals a reference."), + MODULE_NOARGS_ENTRY(test_PyList_SetItem_steals_replace, + "Check that PyList_SetItem() steals a reference on replacement."), + MODULE_NOARGS_ENTRY(test_PyList_SET_ITEM_steals_replace, + "Check that PyList_SET_ITEM() steals a reference on replacement."), + MODULE_NOARGS_ENTRY(test_PyList_SetItem_replace_same, + "Check how PyTuple_SetItem() behaves on replacement of the same value."), + MODULE_NOARGS_ENTRY(test_PyList_SET_ITEM_replace_same, + "Check how PyTuple_SET_ITEM() behaves on replacement of the same value."), + MODULE_NOARGS_ENTRY(test_PyList_SetItem_NULL, "Check that PyList_SetItem() with NULL does not error."), + MODULE_NOARGS_ENTRY(test_PyList_SET_ITEM_NULL, "Check that PyList_SET_ITEM() with NULL does not error."), + MODULE_NOARGS_ENTRY(test_PyList_SetIem_NULL_SetItem, + "Check that PyList_SetItem() with NULL then with an object does not error."), + MODULE_NOARGS_ENTRY(test_PyList_SET_ITEM_NULL_SET_ITEM, + "Check that PyList_SET_ITEM() with NULL then with an object does not error."), + MODULE_NOARGS_ENTRY(test_PyList_SetItem_fails_not_a_list, + "Check that PyList_SET_ITEM() fails when not a tuple."), + MODULE_NOARGS_ENTRY(test_PyList_SetItem_fails_out_of_range, + "Check that PyList_SET_ITEM() fails when index out of range."), + MODULE_NOARGS_ENTRY(test_PyList_Append, + "Check that PyList_Append() increments reference counts."), + MODULE_NOARGS_ENTRY(test_PyList_Append_fails_not_a_list, + "Check that PyList_Append() raises when not a list."), + MODULE_NOARGS_ENTRY(test_PyList_Append_fails_NULL, + "Check that PyList_Append() raises on NULL."), + MODULE_NOARGS_ENTRY(test_PyList_Insert, + "Check that PyList_Insert() increments reference counts."), + MODULE_NOARGS_ENTRY(test_PyList_Insert_Is_Truncated, + "Check that PyList_Insert() truncates index."), + MODULE_NOARGS_ENTRY(test_PyList_Insert_Negative_Index, + "Check that PyList_Insert() with negative index."), + MODULE_NOARGS_ENTRY(test_PyList_Insert_fails_not_a_list, + "Check that PyList_Insert() raises when not a list."), + MODULE_NOARGS_ENTRY(test_PyList_Insert_fails_NULL, + "Check that PyList_Insert() raises on NULL."), + MODULE_NOARGS_ENTRY(test_PyList_Py_BuildValue, + "Check that Py_BuildValue() increments reference counts."), +#pragma mark - Testing Dictionaries + MODULE_NOARGS_ENTRY(test_PyDict_SetItem_increments, "Check that PyDict_SetItem() works as expected."), + MODULE_NOARGS_ENTRY(test_PyDict_SetItem_fails_not_a_dict, + "Check that PyDict_SetItem() fails when not a dictionary."), + MODULE_NOARGS_ENTRY(test_PyDict_SetItem_fails_not_hashable, + "Check that PyDict_SetItem() fails when key is not hashable."), + MODULE_NOARGS_ENTRY(test_PyDict_SetDefault_default_unused, + "Check that PyDict_SetDefault() works when the default is not used."), + MODULE_NOARGS_ENTRY(test_PyDict_SetDefault_default_used, + "Check that PyDict_SetDefault() works when the default not used."), +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + MODULE_NOARGS_ENTRY(test_PyDict_SetDefaultRef_default_unused, + "Check that PyDict_SetDefaultRef() works when the default is not used."), + MODULE_NOARGS_ENTRY(test_PyDict_SetDefaultRef_default_used, + "Check that PyDict_SetDefaultRef() works when the default not used."), +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + + MODULE_NOARGS_ENTRY(test_PyDict_GetItem, + "Checks PyDict_GetItem()."), + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + MODULE_NOARGS_ENTRY(test_PyDict_Pop_key_present, + "Check that PyDict_Pop() works when the key is present."), + MODULE_NOARGS_ENTRY(test_PyDict_Pop_key_absent, + "Check that PyDict_Pop() works when the key is absent."), +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 +#pragma mark - Testing Sets + MODULE_NOARGS_ENTRY(test_PySet_Add, "Check PySet_Add()."), + MODULE_NOARGS_ENTRY(test_PySet_Discard, "Check test_PySet_Discard()."), + MODULE_NOARGS_ENTRY(test_PySet_Pop, "Check PySet_Pop()."), + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cRefCount = { + PyModuleDef_HEAD_INIT, + .m_name = "cRefCount", + .m_doc = "Exploring reference counts.", + .m_size = -1, + .m_methods = module_methods, +}; + +PyMODINIT_FUNC PyInit_cRefCount(void) { + return PyModule_Create(&cRefCount); +} diff --git a/src/cpy/SimpleExample/cFibA.c b/src/cpy/SimpleExample/cFibA.c new file mode 100644 index 0000000..8551fef --- /dev/null +++ b/src/cpy/SimpleExample/cFibA.c @@ -0,0 +1,71 @@ +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +long fibonacci(long index) { + if (index < 2) { + return index; + } + return fibonacci(index - 2) + fibonacci(index - 1); +} + +//long fibonacci(long index) { +// static long *cache = NULL; +// if (!cache) { +// /* FIXME */ +// cache = calloc(1000, sizeof(long)); +// } +// if (index < 2) { +// return index; +// } +// if (!cache[index]) { +// cache[index] = fibonacci(index - 2) + fibonacci(index - 1); +// } +// return cache[index]; +//} + +static PyObject * +py_fibonacci(PyObject *Py_UNUSED(module), PyObject *args) { + long index; + + if (!PyArg_ParseTuple(args, "l", &index)) { + return NULL; + } + long result = fibonacci(index); + return Py_BuildValue("l", result); +} + +//static PyObject * +//py_fibonacci(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { +// long index; +// +// static char *keywords[] = {"index", NULL}; +// if (!PyArg_ParseTupleAndKeywords(args, kwargs, "l", keywords, &index)) { +// return NULL; +// } +// long result = fibonacci(index); +// return Py_BuildValue("l", result); +//} +// + +static PyMethodDef module_methods[] = { + {"fibonacci", + (PyCFunction) py_fibonacci, + METH_VARARGS, + "Returns the Fibonacci value." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cFibA = { + PyModuleDef_HEAD_INIT, + .m_name = "cFibA", + .m_doc = "Fibonacci in C.", + .m_size = -1, + .m_methods = module_methods, +}; + +PyMODINIT_FUNC PyInit_cFibA(void) { + PyObject *m = PyModule_Create(&cFibA); + return m; +} diff --git a/src/cpy/SimpleExample/cFibA.h b/src/cpy/SimpleExample/cFibA.h new file mode 100644 index 0000000..fd7356a --- /dev/null +++ b/src/cpy/SimpleExample/cFibA.h @@ -0,0 +1,10 @@ +// +// Created by Paul Ross on 20/02/2023. +// + +#ifndef PYEXTEXAMPLE_CFIB_H +#define PYEXTEXAMPLE_CFIB_H + +long fibonacci(long index); + +#endif //PYEXTEXAMPLE_CFIB_H diff --git a/src/cpy/SimpleExample/cFibB.c b/src/cpy/SimpleExample/cFibB.c new file mode 100644 index 0000000..531aee3 --- /dev/null +++ b/src/cpy/SimpleExample/cFibB.c @@ -0,0 +1,65 @@ +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +long fibonacci(long index) { + static long *cache = NULL; + if (!cache) { + /* FIXME */ + cache = calloc(1000, sizeof(long)); + } + if (index < 2) { + return index; + } + if (!cache[index]) { + cache[index] = fibonacci(index - 2) + fibonacci(index - 1); + } + return cache[index]; +} + +static PyObject * +py_fibonacci(PyObject *Py_UNUSED(module), PyObject *args) { + long index; + + if (!PyArg_ParseTuple(args, "l", &index)) { + return NULL; + } + long result = fibonacci(index); + return Py_BuildValue("l", result); +} + +//static PyObject * +//py_fibonacci(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { +// long index; +// +// static char *keywords[] = {"index", NULL}; +// if (!PyArg_ParseTupleAndKeywords(args, kwargs, "l", keywords, &index)) { +// return NULL; +// } +// long result = fibonacci(index); +// return Py_BuildValue("l", result); +//} +// + +static PyMethodDef module_methods[] = { + { + "fibonacci", + (PyCFunction) py_fibonacci, + METH_VARARGS, + "Returns the Fibonacci value." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cFibB = { + PyModuleDef_HEAD_INIT, + .m_name = "cFibB", + .m_doc = "Fibonacci in C with cache.", + .m_size = -1, + .m_methods = module_methods, +}; + +PyMODINIT_FUNC PyInit_cFibB(void) { + PyObject *m = PyModule_Create(&cFibB); + return m; +} diff --git a/src/cpy/SimpleExample/pFibA.py b/src/cpy/SimpleExample/pFibA.py new file mode 100644 index 0000000..f33d892 --- /dev/null +++ b/src/cpy/SimpleExample/pFibA.py @@ -0,0 +1,4 @@ +def fibonacci(index: int) -> int: + if index < 2: + return index + return fibonacci(index - 2) + fibonacci(index - 1) diff --git a/src/cpy/SimpleExample/pFibB.py b/src/cpy/SimpleExample/pFibB.py new file mode 100644 index 0000000..b7c3279 --- /dev/null +++ b/src/cpy/SimpleExample/pFibB.py @@ -0,0 +1,7 @@ +import functools + +@functools.cache +def fibonacci(index: int) -> int: + if index < 2: + return index + return fibonacci(index - 2) + fibonacci(index - 1) diff --git a/src/cpy/SimpleExample/timeit_test.py b/src/cpy/SimpleExample/timeit_test.py new file mode 100644 index 0000000..435c67d --- /dev/null +++ b/src/cpy/SimpleExample/timeit_test.py @@ -0,0 +1,52 @@ +import timeit + +index = 32 +number = 20 + +print(f'Index: {index} number of times: {number}') +print('Version A, no cacheing:') +# Use pFibA +ti_py = timeit.timeit(f'pFibA.fibonacci({index})', setup='import pFibA', number=number) +print(f'Python timeit: {ti_py:8.6f}') + +# Use cFibA +ti_c = timeit.timeit(f'cFibA.fibonacci({index})', + setup='from cPyExtPatt.SimpleExample import cFibA', number=number) +print(f' C timeit: {ti_c:8.6f}') + +print( + f'C is {ti_py / ti_c if ti_py > ti_c else ti_c / ti_py:.1f}' + f' times {"FASTER" if ti_py > ti_c else "SLOWER"}.' +) + +print() +print('Version A with Python cache, no C cache:') +# Use pFibB +ti_py = timeit.timeit(f'pFibB.fibonacci({index})', setup='import pFibB', number=number) +print(f'Python timeit: {ti_py:8.6f}') + +# Use cFibB +ti_c = timeit.timeit(f'cFibA.fibonacci({index})', + setup='from cPyExtPatt.SimpleExample import cFibA', number=number) +print(f' C timeit: {ti_c:8.6f}') + +print( + f'C is {ti_py / ti_c if ti_py > ti_c else ti_c / ti_py:.1f}' + f' times {"FASTER" if ti_py > ti_c else "SLOWER"}.' +) + +print() +print('Version B, both are cached:') +# Use pFibB +ti_py = timeit.timeit(f'pFibB.fibonacci({index})', setup='import pFibB', number=number) +print(f'Python timeit: {ti_py:8.6f}') + +# Use cFibB +ti_c = timeit.timeit(f'cFibB.fibonacci({index})', + setup='from cPyExtPatt.SimpleExample import cFibB', number=number) +print(f' C timeit: {ti_c:8.6f}') + +print( + f'C is {ti_py / ti_c if ti_py > ti_c else ti_c / ti_py:.1f}' + f' times {"FASTER" if ti_py > ti_c else "SLOWER"}.' +) diff --git a/src/cpy/StructSequence/cStructSequence.c b/src/cpy/StructSequence/cStructSequence.c new file mode 100644 index 0000000..a619448 --- /dev/null +++ b/src/cpy/StructSequence/cStructSequence.c @@ -0,0 +1,527 @@ +// +// Created by Paul Ross on 28/12/2024. +// +// Example of a Struct Sequence Object (equivalent to a named tuple). +// Documentation: https://docs.python.org/3/c-api/tuple.html#struct-sequence-objects +// Example test case: Modules/_testcapimodule.c test_structseq_newtype_doesnt_leak() +// Fairly complicated example: Modules/posixmodule.c +/** + * TODO: + * + * We should cover named tuples/dataclasses etc.: + * https://docs.python.org/3/c-api/tuple.html#struct-sequence-objects + * + */ + +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +#if 0 +// Example test case: Modules/_testcapimodule.c test_structseq_newtype_doesnt_leak() +static PyObject * +test_structseq_newtype_doesnt_leak(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(args)) +{ + PyStructSequence_Desc descr; + PyStructSequence_Field descr_fields[3]; + + descr_fields[0] = (PyStructSequence_Field){"foo", "foo value"}; + descr_fields[1] = (PyStructSequence_Field){NULL, "some hidden value"}; + descr_fields[2] = (PyStructSequence_Field){0, NULL}; + + descr.name = "_testcapi.test_descr"; + descr.doc = "This is used to test for memory leaks in NewType"; + descr.fields = descr_fields; + descr.n_in_sequence = 1; + + PyTypeObject* structseq_type = PyStructSequence_NewType(&descr); + assert(structseq_type != NULL); + assert(PyType_Check(structseq_type)); + assert(PyType_FastSubclass(structseq_type, Py_TPFLAGS_TUPLE_SUBCLASS)); + Py_DECREF(structseq_type); + + Py_RETURN_NONE; +} + +// Fairly complicated examples in: Modules/posixmodule.c + +// A simple example: + +PyDoc_STRVAR(TerminalSize_docstring, + "A tuple of (columns, lines) for holding terminal window size"); + +static PyStructSequence_Field TerminalSize_fields[] = { + {"columns", "width of the terminal window in characters"}, + {"lines", "height of the terminal window in characters"}, + {NULL, NULL} +}; + +static PyStructSequence_Desc TerminalSize_desc = { + "os.terminal_size", + TerminalSize_docstring, + TerminalSize_fields, + 2, +}; +#endif + +#pragma mark - A basic Named Tuple + +PyDoc_STRVAR( + BasicNT_docstring, + "A basic named tuple type with two fields." +); + +static PyStructSequence_Field BasicNT_fields[] = { + {"field_one", "The first field of the named tuple."}, + {"field_two", "The second field of the named tuple."}, + {NULL, NULL} +}; + +static PyStructSequence_Desc BasicNT_desc = { + "cStructSequence.BasicNT", + BasicNT_docstring, + BasicNT_fields, + 2, +}; + +static PyObject * +BasicNT_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + static PyTypeObject *static_BasicNT_Type = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &field_one, &field_two)) { + return NULL; + } + /* The two fields are PyObjects. If your design is that those arguments should be specific types + * then take the opportunity here to test that they are the expected types. + */ + if (!static_BasicNT_Type) { + static_BasicNT_Type = PyStructSequence_NewType(&BasicNT_desc); + if (!static_BasicNT_Type) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a BasicNT type with PyStructSequence_NewType()" + ); + return NULL; + } + } + PyObject *result = PyStructSequence_New(static_BasicNT_Type); + if (!result) { + PyErr_SetString( + PyExc_MemoryError, + "Can not create a Struct Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created (i.e. its reference count is not increased)." + * So we increment as PyStructSequence_SetItem seals the reference otherwise if the callers arguments + * go out of scope we will/may get undefined behaviour when accessing the named tuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + return result; +} + +#pragma mark - A registered Named Tuple + +PyDoc_STRVAR( + NTRegistered_docstring, + "A named tuple type with two fields that is" + "registered with the cStructSequence module." +); + +static PyStructSequence_Field NTRegistered_fields[] = { + {"field_one", "The first field of the named tuple."}, + {"field_two", "The second field of the named tuple."}, + {NULL, NULL} +}; + +static PyStructSequence_Desc NTRegistered_desc = { + "cStructSequence.NTRegistered", + NTRegistered_docstring, + NTRegistered_fields, + 2, +}; + +#pragma mark A un-registered Named Tuple + +PyDoc_STRVAR( + NTUnRegistered_docstring, + "A named tuple type with two fields that is" + " not registered with the cStructSequence module." +); + +static PyStructSequence_Field NTUnRegistered_fields[] = { + {"field_one", "The first field of the named tuple."}, + {"field_two", "The second field of the named tuple."}, + {NULL, NULL} +}; + +static PyStructSequence_Desc NTUnRegistered_desc = { + "cStructSequence.NTUnRegistered", + NTUnRegistered_docstring, + NTUnRegistered_fields, + 2, +}; + +/* Type initailised dynamically by NTUnRegistered_create(). */ +static PyTypeObject *static_NTUnRegisteredType = NULL; + +/* A function that creates a cStructSequence.NTUnRegistered dynamically. */ +static PyObject * +NTUnRegistered_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", kwlist, &field_one, &field_two)) { + return NULL; + } + /* The two fields are PyObjects. If your design is that those arguments should be specific types + * then take the opportunity here to test that they are the expected types. + */ + /* Initialise the static static_NTUnRegisteredType. + * Note: PyStructSequence_NewType returns a new reference. + */ + if (!static_NTUnRegisteredType) { + static_NTUnRegisteredType = PyStructSequence_NewType(&NTUnRegistered_desc); + if (!static_NTUnRegisteredType) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a type with PyStructSequence_NewType()" + ); + return NULL; + } + } + PyObject *result = PyStructSequence_New(static_NTUnRegisteredType); + if (!result) { + PyErr_SetString( + PyExc_MemoryError, + "Can not create a Struct Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created (i.e. its reference count is not increased)." + * So we increment as PyStructSequence_SetItem seals the reference otherwise if the callers arguments + * go out of scope we will/may get undefined behaviour when accessing the named tuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + return result; +} + +#pragma mark - Example of a C struct to PyStructSequence + +/** + * Representation of a simple transaction. + */ +struct cTransaction { + long id; /* The transaction id. */ + char *reference; /* The transaction reference. */ + double amount; /* The transaction amount. */ +}; + +/** + * An example function that might recover a transaction from within C code, + * possibly a C library. + * In practice this will actually do something more useful that this function does! + * + * @param id The transaction ID. + * @return A struct cTransaction corresponding to the transaction ID. + */ +static struct cTransaction get_transaction(long id) { + struct cTransaction ret = {id, "Some reference.", 42.76}; + return ret; +} + +PyDoc_STRVAR( + cTransaction_docstring, + "Example of a named tuple type representing a transaction created in C." + " The type not registered with the cStructSequence module." +); + +static PyStructSequence_Field cTransaction_fields[] = { + {"id", "The transaction id."}, + {"reference", "The transaction reference."}, + {"amount", "The transaction amount."}, + {NULL, NULL} +}; + +static PyStructSequence_Desc cTransaction_desc = { + "cStructSequence.cTransaction", + cTransaction_docstring, + cTransaction_fields, + 3, +}; + +/* Type initialised dynamically by get__cTransactionType(). */ +static PyTypeObject *static_cTransactionType = NULL; + +static PyTypeObject *get_cTransactionType(void) { + if (static_cTransactionType == NULL) { + static_cTransactionType = PyStructSequence_NewType(&cTransaction_desc); + if (static_cTransactionType == NULL) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a cTransaction type with PyStructSequence_NewType()" + ); + return NULL; + } + } + return static_cTransactionType; +} + +/* A function that creates a cStructSequence.NTUnRegistered dynamically. */ +PyObject * +cTransaction_get(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"id", NULL}; + long id = 0l; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "l", kwlist, &id)) { + return NULL; + } + PyObject *result = PyStructSequence_New(get_cTransactionType()); + if (!result) { + assert(PyErr_Occurred()); + return NULL; + } + + struct cTransaction transaction = get_transaction(id); + PyStructSequence_SetItem(result, 0, PyLong_FromLong(transaction.id)); + PyStructSequence_SetItem(result, 1, PyUnicode_FromString(transaction.reference)); + PyStructSequence_SetItem(result, 2, PyFloat_FromDouble(transaction.amount)); + return result; +} + +#pragma mark - A registered Named Tuple with excess fields + +PyDoc_STRVAR( + ExcessNT_docstring, + "A basic named tuple type with excess fields." +); + +static PyStructSequence_Field ExcessNT_fields[] = { + {"field_one", "The first field of the named tuple."}, + {"field_two", "The second field of the named tuple."}, + {"field_three", "The third field of the named tuple, not available to Python."}, + {NULL, NULL} +}; + +static PyStructSequence_Desc ExcessNT_desc = { + "cStructSequence.ExcessNT", + ExcessNT_docstring, + ExcessNT_fields, + 2, /* Of three fields only two are available to Python. */ +}; + +static PyObject * +ExcessNT_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", "field_three", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + PyObject *field_three = NULL; + /* Type initialised dynamically by get__cTransactionType(). */ + static PyTypeObject *static_ExcessNT_Type = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOO", kwlist, &field_one, &field_two, &field_three)) { + return NULL; + } + /* The three fields are PyObjects. If your design is that those arguments should be specific types + * then take the opportunity here to test that they are the expected types. + */ + if (!static_ExcessNT_Type) { + static_ExcessNT_Type = PyStructSequence_NewType(&ExcessNT_desc); + if (!static_ExcessNT_Type) { + PyErr_SetString( + PyExc_MemoryError, + "Can not initialise a ExcessNT type with PyStructSequence_NewType()" + ); + return NULL; + } + } + PyObject *result = PyStructSequence_New(static_ExcessNT_Type); + if (!result) { + PyErr_SetString( + PyExc_MemoryError, + "Can not create a ExcessNT Struct Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created (i.e. its reference count is not increased)." + * So we increment as PyStructSequence_SetItem seals the reference otherwise if the callers arguments + * go out of scope we will/may get undefined behaviour when accessing the named tuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + Py_INCREF(field_three); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + PyStructSequence_SetItem(result, 2, field_three); + return result; +} + +#pragma mark - A registered Named Tuple with an unnamed field + +/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. + * Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... + * + * This is Python 3.11+ specific code. + * Earlier versions give this compile time error: + * E ImportError: dlopen(...cStructSequence.cpython-310-darwin.so, 0x0002): \ + * symbol not found in flat namespace '_PyStructSequence_UnnamedField' + */ +#if PY_VERSION_HEX >= 0x030B0000 + +static PyStructSequence_Field NTWithUnnamedField_fields[] = { + {"field_one", "The first field of the named tuple."}, + /* Use NULL then replace with PyStructSequence_UnnamedField + * otherwise get an error "initializer element is not a compile-time constant" */ + {"field_two", "The second field of the named tuple, not available to Python."}, + {NULL, "Documentation for an unnamed field."}, + {NULL, NULL} +}; + +PyDoc_STRVAR( + NTWithUnnamedField_docstring, + "A basic named tuple type with an unnamed field." +); + +static PyStructSequence_Desc NTWithUnnamedField_desc = { + "cStructSequence.NTWithUnnamedField", + NTWithUnnamedField_docstring, + NTWithUnnamedField_fields, + 1, /* Of three fields only one is available to Python by name. */ +}; + +static PyTypeObject *static_NTWithUnnamedField_Type = NULL; + +/** + * Initialises and returns the \c NTWithUnnamedField_Type. + * @return The initialised type. + */ +static PyTypeObject *get_NTWithUnnamedField_Type(void) { + if (!static_NTWithUnnamedField_Type) { + /* Substitute PyStructSequence_UnnamedField for NULL. */ + NTWithUnnamedField_fields[1].name = PyStructSequence_UnnamedField; + /* Create and initialise the type. */ + static_NTWithUnnamedField_Type = PyStructSequence_NewType(&NTWithUnnamedField_desc); + if (!static_NTWithUnnamedField_Type) { + PyErr_SetString( + PyExc_RuntimeError, + "Can not initialise a NTWithUnnamedField type with PyStructSequence_NewType()" + ); + return NULL; + } + } + return static_NTWithUnnamedField_Type; +} + +static PyObject * +NTWithUnnamedField_create(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwds) { + assert(!PyErr_Occurred()); + static char *kwlist[] = {"field_one", "field_two", "field_three", NULL}; + PyObject *field_one = NULL; + PyObject *field_two = NULL; + PyObject *field_three = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOO", kwlist, &field_one, &field_two, &field_three)) { + return NULL; + } + /* The three fields are PyObjects. If your design is that those arguments should be specific types + * then take the opportunity here to test that they are the expected types. + */ + + PyObject *result = PyStructSequence_New(get_NTWithUnnamedField_Type()); + if (!result) { + PyErr_SetString( + PyExc_RuntimeError, + "Can not create a NTWithUnnamedField Struct Sequence with PyStructSequence_New()" + ); + return NULL; + } + /* PyArg_ParseTupleAndKeywords with "O" gives a borrowed reference. + * https://docs.python.org/3/c-api/arg.html#other-objects + * "A new strong reference to the object is not created (i.e. its reference count is not increased)." + * So we increment as PyStructSequence_SetItem seals the reference otherwise if the callers arguments + * go out of scope we will/may get undefined behaviour when accessing the named tuple fields. + */ + Py_INCREF(field_one); + Py_INCREF(field_two); + Py_INCREF(field_three); + PyStructSequence_SetItem(result, 0, field_one); + PyStructSequence_SetItem(result, 1, field_two); + PyStructSequence_SetItem(result, 2, field_three); + assert(!PyErr_Occurred()); + return result; +} +#endif + +#pragma mark - cStructSequence module methods + +static PyMethodDef cStructSequence_methods[] = { + {"BasicNT_create", (PyCFunction) BasicNT_create, METH_VARARGS | METH_KEYWORDS, + "Create a BasicNT from the given values."}, + {"NTUnRegistered_create", (PyCFunction) NTUnRegistered_create, METH_VARARGS | METH_KEYWORDS, + "Create a NTUnRegistered from the given values."}, + {"cTransaction_get", (PyCFunction) cTransaction_get, METH_VARARGS | METH_KEYWORDS, + "Example of getting a transaction."}, + {"ExcessNT_create", (PyCFunction) ExcessNT_create, METH_VARARGS | METH_KEYWORDS, + "Create a ExcessNT from the given values."}, +/* Python 3.11+ specific code. + * Earlier versions give this compile time error: + * E ImportError: dlopen(...cStructSequence.cpython-310-darwin.so, 0x0002): \ + * symbol not found in flat namespace '_PyStructSequence_UnnamedField' + */ +#if PY_VERSION_HEX >= 0x030B0000 + {"NTWithUnnamedField_create", (PyCFunction) NTWithUnnamedField_create, METH_VARARGS | METH_KEYWORDS, + "Create a NTWithUnnamedField from the given values."}, +#endif + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cStructSequence_cmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cStructSequence", + .m_doc = ( + "Example module that works with Struct Sequence (named tuple) objects." + ), + .m_size = -1, + .m_methods = cStructSequence_methods, +}; + +PyMODINIT_FUNC +PyInit_cStructSequence(void) { + PyObject *m; + m = PyModule_Create(&cStructSequence_cmodule); + if (m == NULL) { + return NULL; + } + /* Initialise NTRegisteredType */ + PyObject *NTRegisteredType = (PyObject *) PyStructSequence_NewType(&NTRegistered_desc); + if (NTRegisteredType == NULL) { + Py_DECREF(m); + return NULL; + } + Py_INCREF(NTRegisteredType); + PyModule_AddObject(m, "NTRegisteredType", NTRegisteredType); + + return m; +} + diff --git a/src/cpy/SubClass/sublist.c b/src/cpy/SubClass/sublist.c new file mode 100644 index 0000000..edd0aed --- /dev/null +++ b/src/cpy/SubClass/sublist.c @@ -0,0 +1,105 @@ +// +// sublist.c +// Subclassing a Python list. +// +// Created by Paul Ross on 22/07/2024. +// Copyright (c) 2024 Paul Ross. All rights reserved. +// +// Based on: https://docs.python.org/3/extending/newtypes_tutorial.html#subclassing-other-types +// That describes sub-classing a list. +// However as well as the increment function this counts how many times +// append() is called and uses the super() class to call the base class append. + +#define PY_SSIZE_T_CLEAN +#include +#include "structmember.h" + +#include "py_call_super.h" + +typedef struct { + PyListObject list; + int state; + int appends; +} SubListObject; + +static int +SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) { + return -1; + } + self->state = 0; + self->appends = 0; + return 0; +} + +static PyObject * +SubList_increment(SubListObject *self, PyObject *Py_UNUSED(unused)) { + self->state++; + return PyLong_FromLong(self->state); +} + +static PyObject * +SubList_append(SubListObject *self, PyObject *args) { + PyObject *result = call_super_name((PyObject *)self, "append", + args, NULL); + if (result) { + self->appends++; + } + return result; +} + + +static PyMethodDef SubList_methods[] = { + {"increment", (PyCFunction) SubList_increment, METH_NOARGS, + PyDoc_STR("increment state counter")}, + {"append", (PyCFunction) SubList_append, METH_VARARGS, + PyDoc_STR("append an item")}, + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef SubList_members[] = { + {"state", T_INT, offsetof(SubListObject, state), 0, + "Value of the state."}, + {"appends", T_INT, offsetof(SubListObject, appends), 0, + "Number of append operations."}, + {NULL, 0, 0, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject SubListType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "sublist.SubList", + .tp_doc = PyDoc_STR("SubList objects"), + .tp_basicsize = sizeof(SubListObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_init = (initproc) SubList_init, + .tp_methods = SubList_methods, + .tp_members = SubList_members, +}; + +static PyModuleDef sublistmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "sublist", + .m_doc = "Module that contains a subclass of a list.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_sublist(void) { + PyObject *m; + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return NULL; + } + m = PyModule_Create(&sublistmodule); + if (m == NULL) { + return NULL; + } + Py_INCREF(&SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/Threads/cThreadLock.h b/src/cpy/Threads/cThreadLock.h new file mode 100644 index 0000000..863f5d9 --- /dev/null +++ b/src/cpy/Threads/cThreadLock.h @@ -0,0 +1,67 @@ +// +// Created by Paul Ross on 22/07/2024. +// + +#ifndef PYTHONEXTENSIONPATTERNS_CTHREADLOCK_H +#define PYTHONEXTENSIONPATTERNS_CTHREADLOCK_H + +#include +#include "structmember.h" + +#ifdef WITH_THREAD +#include "pythread.h" +#endif + +#ifdef WITH_THREAD +/* A RAII wrapper around the PyThread_type_lock. */ +template +class AcquireLock { +public: + AcquireLock(T *pObject) : m_pObject(pObject) { + assert(m_pObject); + assert(m_pObject->lock); + Py_INCREF(m_pObject); + if (!PyThread_acquire_lock(m_pObject->lock, NOWAIT_LOCK)) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(m_pObject->lock, WAIT_LOCK); + Py_END_ALLOW_THREADS + } + } + ~AcquireLock() { + assert(m_pObject); + assert(m_pObject->lock); + PyThread_release_lock(m_pObject->lock); + Py_DECREF(m_pObject); + } +private: + T *m_pObject; +}; + +#else +/* Make the class a NOP which should get optimised out. */ +template +class AcquireLock { +public: + AcquireLock(T *) {} +}; +#endif + +// From https://github.com/python/cpython/blob/main/Modules/_bz2module.c +// #define ACQUIRE_LOCK(obj) do { \ +// if (!PyThread_acquire_lock((obj)->lock, 0)) { \ +// Py_BEGIN_ALLOW_THREADS \ +// PyThread_acquire_lock((obj)->lock, 1); \ +// Py_END_ALLOW_THREADS \ +// } } while (0) +//#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) + +// /Library/Frameworks/Python.framework/Versions/3.11/include/python3.11/ceval.h +// #define Py_BEGIN_ALLOW_THREADS { \ +// PyThreadState *_save; \ +// _save = PyEval_SaveThread(); +//#define Py_BLOCK_THREADS PyEval_RestoreThread(_save); +//#define Py_UNBLOCK_THREADS _save = PyEval_SaveThread(); +//#define Py_END_ALLOW_THREADS PyEval_RestoreThread(_save); \ +// } + +#endif //PYTHONEXTENSIONPATTERNS_CTHREADLOCK_H diff --git a/src/cpy/Threads/cppsublist.cpp b/src/cpy/Threads/cppsublist.cpp new file mode 100644 index 0000000..e146506 --- /dev/null +++ b/src/cpy/Threads/cppsublist.cpp @@ -0,0 +1,166 @@ +// +// cppsublist.cpp +// Subclassing a Python list. +// +// Created by Paul Ross on 22/07/2024. +// Copyright (c) 2024 Paul Ross. All rights reserved. +// +// Based on: https://docs.python.org/3/extending/newtypes_tutorial.html#subclassing-other-types +// +// This is very like src/cpy/SubClass/sublist.c but it includes a slow max() method +// to illustrate thread contention. +// So it needs a thread lock. + +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +#include "py_call_super.h" +#include "cThreadLock.h" +#include + + +typedef struct { + PyListObject list; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} SubListObject; + +static int +SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) { + return -1; + } +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate thread lock."); + return -2; + } +#endif + return 0; +} + +static void +SubList_dealloc(SubListObject *self) { + /* Deallocate other fields here. */ +#ifdef WITH_THREAD + if (self->lock) { + PyThread_free_lock(self->lock); + self->lock = NULL; + } +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +void sleep_milliseconds(long ms) { + struct timespec tim_request, tim_remain; + tim_request.tv_sec = 0; + tim_request.tv_nsec = ms * 1000L * 1000L; + nanosleep(&tim_request, &tim_remain); +} + +/** append with a thread lock. */ +static PyObject * +SubList_append(SubListObject *self, PyObject *args) { + AcquireLock local_lock((SubListObject *)self); + PyObject *result = call_super_name( + (PyObject *) self, "append", args, NULL + ); + // 0.25s delay to demonstrate holding on to the thread. + sleep_milliseconds(250L); + return result; +} + +/** This is a deliberately laborious find of the maximum value to + * demonstrate protection against thread contention. + */ +static PyObject * +SubList_max(PyObject *self, PyObject *Py_UNUSED(unused)) { + assert(!PyErr_Occurred()); + AcquireLock local_lock((SubListObject *)self); + PyObject *ret = NULL; + // SubListObject + size_t length = PyList_Size(self); + if (length == 0) { + // Raise + PyErr_SetString(PyExc_ValueError, "max() on empty list."); + } else { + // Return first + ret = PyList_GetItem(self, 0); + if (length > 1) { + // laborious compare + PyObject *item = NULL; + for(Py_ssize_t i = 1; i 0) { + ret = item; + } + // 2ms delay to demonstrate holding on to the thread. + sleep_milliseconds(2L); + } + } + Py_INCREF(ret); + } +// // 0.25s delay to demonstrate holding on to the thread. +// sleep_milliseconds(250L); + return ret; +} + +static PyMethodDef SubList_methods[] = { + {"append", (PyCFunction) SubList_append, METH_VARARGS, + PyDoc_STR("append an item with sleep(1).")}, + {"max", (PyCFunction) SubList_max, METH_NOARGS, + PyDoc_STR("Return the maximum value with sleep(1).")}, + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef SubList_members[] = { + {NULL, 0, 0, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject cppSubListType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "cppsublist.cppSubList", + .tp_basicsize = sizeof(SubListObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = PyDoc_STR("C++ SubList object"), + .tp_methods = SubList_methods, + .tp_members = SubList_members, + .tp_init = (initproc) SubList_init, + .tp_dealloc = (destructor) SubList_dealloc, +}; + +static PyModuleDef cppsublistmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "cppsublist", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_cppsublist(void) { + PyObject * m; + cppSubListType.tp_base = &PyList_Type; + if (PyType_Ready(&cppSubListType) < 0) { + return NULL; + } + m = PyModule_Create(&cppsublistmodule); + if (m == NULL) { + return NULL; + } + Py_INCREF(&cppSubListType); + if (PyModule_AddObject(m, "cppSubList", (PyObject *) &cppSubListType) < 0) { + Py_DECREF(&cppSubListType); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/Threads/csublist.c b/src/cpy/Threads/csublist.c new file mode 100644 index 0000000..88be796 --- /dev/null +++ b/src/cpy/Threads/csublist.c @@ -0,0 +1,175 @@ +// +// csublist.c +// Subclassing a Python list. +// +// Created by Paul Ross on 22/07/2024. +// Copyright (c) 2024 Paul Ross. All rights reserved. +// +// Based on: https://docs.python.org/3/extending/newtypes_tutorial.html#subclassing-other-types +// +// This is very like src/cpy/SubClass/sublist.c but it includes a slow max() method +// to illustrate thread contention. +// So it needs a thread lock. + +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +#include "py_call_super.h" +#include + +// From https://github.com/python/cpython/blob/main/Modules/_bz2module.c +#define ACQUIRE_LOCK(obj) do { \ + if (!PyThread_acquire_lock((obj)->lock, 0)) { \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((obj)->lock, 1); \ + Py_END_ALLOW_THREADS \ + } } while (0) +#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) + +typedef struct { + PyListObject list; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} SubListObject; + +static int +SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) { + return -1; + } +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate thread lock."); + return -2; + } +#endif + return 0; +} + +static void +SubList_dealloc(SubListObject *self) { + /* Deallocate other fields here. */ +#ifdef WITH_THREAD + if (self->lock) { + PyThread_free_lock(self->lock); + self->lock = NULL; + } +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +void sleep_milliseconds(long ms) { + struct timespec tim_request, tim_remain; + tim_request.tv_sec = 0; + tim_request.tv_nsec = ms * 1000L * 1000L; + nanosleep(&tim_request, &tim_remain); +} + +/** append with a thread lock. */ +static PyObject * +SubList_append(SubListObject *self, PyObject *args) { + ACQUIRE_LOCK(self); + PyObject *result = call_super_name( + (PyObject *) self, "append", args, NULL + ); + // 0.25s delay to demonstrate holding on to the thread. + sleep_milliseconds(250L); + RELEASE_LOCK(self); + return result; +} + +/** This is a deliberately laborious find of the maximum value to + * demonstrate protection against thread contention. + */ +static PyObject * +SubList_max(PyObject *self, PyObject *Py_UNUSED(unused)) { + assert(!PyErr_Occurred()); + ACQUIRE_LOCK((SubListObject *)self); + PyObject *ret = NULL; + // SubListObject + size_t length = PyList_Size(self); + if (length == 0) { + // Raise + PyErr_SetString(PyExc_ValueError, "max() on empty list."); + } else { + // Return first + ret = PyList_GetItem(self, 0); + if (length > 1) { + // Laborious compare + PyObject *item = NULL; + for(Py_ssize_t i = 1; i 0) { + ret = item; + } + // 2ms delay to demonstrate holding on to the thread. + sleep_milliseconds(2L); + } + } + Py_INCREF(ret); + } +// // 0.25s delay to demonstrate holding on to the thread. +// sleep_milliseconds(250L); + RELEASE_LOCK((SubListObject *)self); + return ret; +} + +static PyMethodDef SubList_methods[] = { + {"append", (PyCFunction) SubList_append, METH_VARARGS, + PyDoc_STR("append an item with sleep().")}, + {"max", (PyCFunction) SubList_max, METH_NOARGS, + PyDoc_STR("Return the maximum value with sleep(1).")}, + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef SubList_members[] = { + {NULL, 0, 0, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject SubListType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "csublist.cSubList", + .tp_basicsize = sizeof(SubListObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = PyDoc_STR("SubList objects"), + .tp_methods = SubList_methods, + .tp_members = SubList_members, + .tp_init = (initproc) SubList_init, + .tp_dealloc = (destructor) SubList_dealloc, +}; + +static PyModuleDef csublistmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "csublist", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_csublist(void) { + PyObject * m; + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return NULL; + } + m = PyModule_Create(&csublistmodule); + if (m == NULL) { + return NULL; + } + Py_INCREF(&SubListType); + if (PyModule_AddObject(m, "cSubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/Util/py_call_super.c b/src/cpy/Util/py_call_super.c new file mode 100644 index 0000000..d3a62ea --- /dev/null +++ b/src/cpy/Util/py_call_super.c @@ -0,0 +1,211 @@ +// +// py_call_super.c +// PythonSubclassList +// +// Provides C functions to call the Python super() class. +// +// Created by Paul Ross on 03/05/2016. +// Copyright (c) 2016-2024 Paul Ross. All rights reserved. +// + +#include "py_call_super.h" + +//#ifdef __cplusplus +//extern "C" +//{ + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation creates a new super object on each call. + */ +PyObject * +call_super_pyname(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *super_func = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + PyObject *result = NULL; + + // Error check input + if (!PyUnicode_Check(func_name)) { + PyErr_Format(PyExc_TypeError, + "super() must be called with unicode attribute not %s", + Py_TYPE(func_name)->tp_name); + } + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + if (!super_args) { + Py_DECREF(self->ob_type); + Py_DECREF(self); + PyErr_SetString(PyExc_RuntimeError, "Could not create arguments for super()."); + goto except; + } + super_func = PyType_GenericNew(&PySuper_Type, super_args, NULL); + if (!super_func) { + PyErr_SetString(PyExc_RuntimeError, "Could not create super()."); + goto except; + } + // Use tuple as first arg, super() second arg (i.e. kwargs) should be NULL + super_func->ob_type->tp_init(super_func, super_args, NULL); + if (PyErr_Occurred()) { + goto except; + } + func = PyObject_GetAttr(super_func, func_name); + if (!func) { + assert(PyErr_Occurred()); + goto except; + } + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_AttributeError, + "super() attribute \"%S\" is not callable.", func_name); + goto except; + } + result = PyObject_Call(func, args, kwargs); + if (!result) { + assert(PyErr_Occurred()); + goto except; + } + assert(!PyErr_Occurred()); + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(result); + result = NULL; +finally: + Py_XDECREF(super_func); + Py_XDECREF(super_args); + Py_XDECREF(func); + return result; +} + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a C string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_name(PyObject *self, const char *func_cname, + PyObject *args, PyObject *kwargs) { + PyObject *result = NULL; + PyObject *func_name = PyUnicode_FromFormat(func_cname); + if (!func_name) { + PyErr_SetString(PyExc_RuntimeError, + "call_super_name(): Could not create string."); + return NULL; + } + result = call_super_pyname(self, func_name, args, kwargs); + Py_DECREF(func_name); + return result; +} + + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_pyname_lookup(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *result = NULL; + PyObject *builtins = NULL; + PyObject *super_type = NULL; + PyObject *super = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + + builtins = PyImport_AddModule("builtins"); + if (!builtins) { + assert(PyErr_Occurred()); + goto except; + } + // Borrowed reference + Py_INCREF(builtins); + super_type = PyObject_GetAttrString(builtins, "super"); + if (!super_type) { + assert(PyErr_Occurred()); + goto except; + } + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + if (!super_args) { + Py_DECREF(self->ob_type); + Py_DECREF(self); + PyErr_SetString(PyExc_RuntimeError, "Could not create arguments for super()."); + goto except; + } + super = PyObject_Call(super_type, super_args, NULL); + if (!super) { + assert(PyErr_Occurred()); + goto except; + } + // The following code is the same as call_super_pyname() + func = PyObject_GetAttr(super, func_name); + if (!func) { + assert(PyErr_Occurred()); + goto except; + } + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_AttributeError, + "super() attribute \"%S\" is not callable.", func_name); + goto except; + } + result = PyObject_Call(func, args, kwargs); + if (!result) { + assert(PyErr_Occurred()); + goto except; + } + assert(!PyErr_Occurred()); + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(result); + result = NULL; +finally: + Py_XDECREF(builtins); + Py_XDECREF(super_args); + Py_XDECREF(super_type); + Py_XDECREF(super); + Py_XDECREF(func); + return result; +} + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a C string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_name_lookup(PyObject *self, const char *func_cname, + PyObject *args, PyObject *kwargs) { + + PyObject *result = NULL; + PyObject *func_name = PyUnicode_FromFormat(func_cname); + if (!func_name) { + PyErr_SetString(PyExc_RuntimeError, + "call_super_name_lookup(): Could not create string."); + return NULL; + } + result = call_super_pyname_lookup(self, func_name, args, kwargs); + Py_DECREF(func_name); + return result; +} +//} +//#endif /* __cplusplus */ diff --git a/src/cpy/Util/py_call_super.cpp b/src/cpy/Util/py_call_super.cpp new file mode 100644 index 0000000..d3a62ea --- /dev/null +++ b/src/cpy/Util/py_call_super.cpp @@ -0,0 +1,211 @@ +// +// py_call_super.c +// PythonSubclassList +// +// Provides C functions to call the Python super() class. +// +// Created by Paul Ross on 03/05/2016. +// Copyright (c) 2016-2024 Paul Ross. All rights reserved. +// + +#include "py_call_super.h" + +//#ifdef __cplusplus +//extern "C" +//{ + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation creates a new super object on each call. + */ +PyObject * +call_super_pyname(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *super_func = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + PyObject *result = NULL; + + // Error check input + if (!PyUnicode_Check(func_name)) { + PyErr_Format(PyExc_TypeError, + "super() must be called with unicode attribute not %s", + Py_TYPE(func_name)->tp_name); + } + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + if (!super_args) { + Py_DECREF(self->ob_type); + Py_DECREF(self); + PyErr_SetString(PyExc_RuntimeError, "Could not create arguments for super()."); + goto except; + } + super_func = PyType_GenericNew(&PySuper_Type, super_args, NULL); + if (!super_func) { + PyErr_SetString(PyExc_RuntimeError, "Could not create super()."); + goto except; + } + // Use tuple as first arg, super() second arg (i.e. kwargs) should be NULL + super_func->ob_type->tp_init(super_func, super_args, NULL); + if (PyErr_Occurred()) { + goto except; + } + func = PyObject_GetAttr(super_func, func_name); + if (!func) { + assert(PyErr_Occurred()); + goto except; + } + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_AttributeError, + "super() attribute \"%S\" is not callable.", func_name); + goto except; + } + result = PyObject_Call(func, args, kwargs); + if (!result) { + assert(PyErr_Occurred()); + goto except; + } + assert(!PyErr_Occurred()); + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(result); + result = NULL; +finally: + Py_XDECREF(super_func); + Py_XDECREF(super_args); + Py_XDECREF(func); + return result; +} + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a C string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_name(PyObject *self, const char *func_cname, + PyObject *args, PyObject *kwargs) { + PyObject *result = NULL; + PyObject *func_name = PyUnicode_FromFormat(func_cname); + if (!func_name) { + PyErr_SetString(PyExc_RuntimeError, + "call_super_name(): Could not create string."); + return NULL; + } + result = call_super_pyname(self, func_name, args, kwargs); + Py_DECREF(func_name); + return result; +} + + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a Python string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_pyname_lookup(PyObject *self, PyObject *func_name, + PyObject *args, PyObject *kwargs) { + PyObject *result = NULL; + PyObject *builtins = NULL; + PyObject *super_type = NULL; + PyObject *super = NULL; + PyObject *super_args = NULL; + PyObject *func = NULL; + + builtins = PyImport_AddModule("builtins"); + if (!builtins) { + assert(PyErr_Occurred()); + goto except; + } + // Borrowed reference + Py_INCREF(builtins); + super_type = PyObject_GetAttrString(builtins, "super"); + if (!super_type) { + assert(PyErr_Occurred()); + goto except; + } + // Will be decremented when super_args is decremented if Py_BuildValue succeeds. + Py_INCREF(self->ob_type); + Py_INCREF(self); + super_args = Py_BuildValue("OO", (PyObject *) self->ob_type, self); + if (!super_args) { + Py_DECREF(self->ob_type); + Py_DECREF(self); + PyErr_SetString(PyExc_RuntimeError, "Could not create arguments for super()."); + goto except; + } + super = PyObject_Call(super_type, super_args, NULL); + if (!super) { + assert(PyErr_Occurred()); + goto except; + } + // The following code is the same as call_super_pyname() + func = PyObject_GetAttr(super, func_name); + if (!func) { + assert(PyErr_Occurred()); + goto except; + } + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_AttributeError, + "super() attribute \"%S\" is not callable.", func_name); + goto except; + } + result = PyObject_Call(func, args, kwargs); + if (!result) { + assert(PyErr_Occurred()); + goto except; + } + assert(!PyErr_Occurred()); + goto finally; +except: + assert(PyErr_Occurred()); + Py_XDECREF(result); + result = NULL; +finally: + Py_XDECREF(builtins); + Py_XDECREF(super_args); + Py_XDECREF(super_type); + Py_XDECREF(super); + Py_XDECREF(func); + return result; +} + +/* Call func_name on the super classes of self with the arguments and + * keyword arguments. + * + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * + * func_name is a C string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_name_lookup(PyObject *self, const char *func_cname, + PyObject *args, PyObject *kwargs) { + + PyObject *result = NULL; + PyObject *func_name = PyUnicode_FromFormat(func_cname); + if (!func_name) { + PyErr_SetString(PyExc_RuntimeError, + "call_super_name_lookup(): Could not create string."); + return NULL; + } + result = call_super_pyname_lookup(self, func_name, args, kwargs); + Py_DECREF(func_name); + return result; +} +//} +//#endif /* __cplusplus */ diff --git a/src/cpy/Util/py_call_super.h b/src/cpy/Util/py_call_super.h new file mode 100644 index 0000000..e72ab26 --- /dev/null +++ b/src/cpy/Util/py_call_super.h @@ -0,0 +1,48 @@ +// +// py_call_super.h +// PythonSubclassList +// +// Provides C functions to call the Python super() class. +// +// Created by Paul Ross on 03/05/2016. +// Copyright (c) 2016 Paul Ross. All rights reserved. +// + +#ifndef __UTIL_PY_CALL_SUPER__ +#define __UTIL_PY_CALL_SUPER__ + +#include + +/* Call func_name on the super classes of self with the arguments and keyword arguments. + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * func_name is a Python string. + * The implementation creates a new super object on each call. + */ +PyObject * +call_super_pyname(PyObject *self, PyObject *func_name, PyObject *args, PyObject *kwargs); + +/* Call func_name on the super classes of self with the arguments and keyword arguments. + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * func_name is a C string. + * The implementation creates a new super object on each call. + */ +PyObject * +call_super_name(PyObject *self, const char *func_cname, PyObject *args, PyObject *kwargs); + +/* Call func_name on the super classes of self with the arguments and keyword arguments. + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * func_name is a Python string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_pyname_lookup(PyObject *self, PyObject *func_name, PyObject *args, PyObject *kwargs); + +/* Call func_name on the super classes of self with the arguments and keyword arguments. + * Equivalent to getattr(super(type(self), self), func_name)(*args, **kwargs) + * func_name is a C string. + * The implementation uses the builtin super(). + */ +PyObject * +call_super_name_lookup(PyObject *self, const char *func_cname, PyObject *args, PyObject *kwargs); + +#endif /* #ifndef __UTIL_PY_CALL_SUPER__ */ diff --git a/src/cpy/Watchers/DictWatcher.c b/src/cpy/Watchers/DictWatcher.c new file mode 100644 index 0000000..94eec1b --- /dev/null +++ b/src/cpy/Watchers/DictWatcher.c @@ -0,0 +1,385 @@ +// +// Created by Paul Ross on 30/01/2025. +// +//#define PPY_SSIZE_T_CLEAN +// +//#include "Python.h" + +#include "DictWatcher.h" +#include "pyextpatt_util.h" + +/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2 + * Therefore 0x030C0000 == 3.12.0 + */ +#if PY_VERSION_HEX < 0x030C0000 + +#error "Required version of Python is 3.12+ (PY_VERSION_HEX >= 0x030C0000)" + +#else + +// Event counters for a dictionary +static long static_dict_added = 0L; +static long static_dict_modified = 0L; +static long static_dict_deleted = 0L; +static long static_dict_cloned = 0L; +static long static_dict_cleared = 0L; +static long static_dict_deallocated = 0L; + +#define GET_STATIC_DICT_VALUE(name) \ + long get_##name(void) { \ + return name; \ + } \ + + +GET_STATIC_DICT_VALUE(static_dict_added) + +GET_STATIC_DICT_VALUE(static_dict_modified) + +GET_STATIC_DICT_VALUE(static_dict_deleted) + +GET_STATIC_DICT_VALUE(static_dict_cloned) + +GET_STATIC_DICT_VALUE(static_dict_cleared) + +GET_STATIC_DICT_VALUE(static_dict_deallocated) + + +// Dictionary callback function +static int dict_watcher_inc_event_counter(PyDict_WatchEvent event, PyObject *Py_UNUSED(dict), PyObject *Py_UNUSED(key), + PyObject *Py_UNUSED(new_value)) { + switch (event) { + case PyDict_EVENT_ADDED: + static_dict_added++; + break; + case PyDict_EVENT_MODIFIED: + static_dict_modified++; + break; + case PyDict_EVENT_DELETED: + static_dict_deleted++; + break; + case PyDict_EVENT_CLONED: + static_dict_cloned++; + break; + case PyDict_EVENT_CLEARED: + static_dict_cleared++; + break; + case PyDict_EVENT_DEALLOCATED: + static_dict_deallocated++; + break; + default: + Py_UNREACHABLE(); + break; + } + /* No exception set. */ + return 0; +} + +/** + * Create a dictionary, register the callback and exercise it by adding a single key and value. + */ +void dbg_PyDict_EVENT_ADDED(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count = 0; + int api_ret_val = 0; + long event_value_previous = 0L; + long event_value_current = 0L; + + PyObject *container = PyDict_New(); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + // Set watcher. + int watcher_id = PyDict_AddWatcher(&dict_watcher_inc_event_counter); + api_ret_val = PyDict_Watch(watcher_id, container); + assert(api_ret_val == 0); + // Now add a key/value + event_value_previous = get_static_dict_added(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *val = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(val); + assert(ref_count == 1); + api_ret_val = PyDict_SetItem(container, key, val); + assert(api_ret_val == 0); + // Check result + event_value_current = get_static_dict_added(); + assert(event_value_current == event_value_previous + 1); + // Clean up. + api_ret_val = PyDict_Unwatch(watcher_id, container); + assert(api_ret_val == 0); + api_ret_val = PyDict_ClearWatcher(watcher_id); + assert(api_ret_val == 0); + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(val); +} + +/** + * Create a dictionary, register the callback and exercise it by adding a single key and value then replacing that + * value with another. + */ +void dbg_PyDict_EVENT_MODIFIED(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count = 0; + int api_ret_val = 0; + long event_value_added_previous = 0L; + long event_value_added_current = 0L; + long event_value_modified_previous = 0L; + long event_value_modified_current = 0L; + + PyObject *container = PyDict_New(); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + // Set watcher. + int watcher_id = PyDict_AddWatcher(&dict_watcher_inc_event_counter); + api_ret_val = PyDict_Watch(watcher_id, container); + assert(api_ret_val == 0); + // Now add a key/value + event_value_added_previous = get_static_dict_added(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *val_a = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(val_a); + assert(ref_count == 1); + api_ret_val = PyDict_SetItem(container, key, val_a); + assert(api_ret_val == 0); + // Check result + event_value_added_current = get_static_dict_added(); + assert(event_value_added_current == event_value_added_previous + 1); + // Now modify the dictionary by resetting the same value and check the modified counter. + PyObject *val_b = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(val_b); + assert(ref_count == 1); + event_value_modified_previous = get_static_dict_modified(); + api_ret_val = PyDict_SetItem(container, key, val_b); + assert(api_ret_val == 0); + event_value_modified_current = get_static_dict_modified(); + assert(event_value_modified_current == event_value_modified_previous + 1); + // Clean up. + api_ret_val = PyDict_Unwatch(watcher_id, container); + assert(api_ret_val == 0); + api_ret_val = PyDict_ClearWatcher(watcher_id); + assert(api_ret_val == 0); + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(val_a); + Py_DECREF(val_b); +} + +/** + * Demonstrates that replacement with the same value does not generate an PyDict_EVENT_MODIFIED event. + */ +void dbg_PyDict_EVENT_MODIFIED_same_value_no_event(void) { + printf("%s():\n", __FUNCTION__); + if (PyErr_Occurred()) { + fprintf(stderr, "%s(): On entry PyErr_Print() %s#%d:\n", __FUNCTION__, __FILE_NAME__, __LINE__); + PyErr_Print(); + return; + } + assert(!PyErr_Occurred()); + int ref_count = 0; + int api_ret_val = 0; + long event_value_added_previous = 0L; + long event_value_added_current = 0L; + long event_value_modified_previous = 0L; + long event_value_modified_current = 0L; + + PyObject *container = PyDict_New(); + assert(container); + ref_count = Py_REFCNT(container); + assert(ref_count == 1); + // Set watcher. + int watcher_id = PyDict_AddWatcher(&dict_watcher_inc_event_counter); + api_ret_val = PyDict_Watch(watcher_id, container); + assert(api_ret_val == 0); + // Now add a key/value + event_value_added_previous = get_static_dict_added(); + PyObject *key = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(key); + assert(ref_count == 1); + PyObject *val = new_unique_string(__FUNCTION__, NULL); + ref_count = Py_REFCNT(val); + assert(ref_count == 1); + api_ret_val = PyDict_SetItem(container, key, val); + assert(api_ret_val == 0); + // Check result + event_value_added_current = get_static_dict_added(); + assert(event_value_added_current == event_value_added_previous + 1); + // Now modify the dictionary by resetting the same value. + event_value_modified_previous = static_dict_modified; + api_ret_val = PyDict_SetItem(container, key, val); + assert(api_ret_val == 0); + event_value_modified_current = static_dict_modified; + assert(event_value_modified_current == event_value_modified_previous + 0); + // Clean up. + api_ret_val = PyDict_Unwatch(watcher_id, container); + assert(api_ret_val == 0); + api_ret_val = PyDict_ClearWatcher(watcher_id); + assert(api_ret_val == 0); + Py_DECREF(container); + Py_DECREF(key); + Py_DECREF(val); +} + +#pragma mark Verbose watcher to report Python file/line + +/** NOTE: This is based on pymemtrace code. */ + +static const unsigned char MT_U_STRING[] = ""; +static const char MT_STRING[] = ""; + +static const unsigned char * +get_python_file_name(PyFrameObject *frame) { + if (frame) { +#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + /* See https://docs.python.org/3.11/whatsnew/3.11.html#pyframeobject-3-11-hiding */ + const unsigned char *file_name = PyUnicode_1BYTE_DATA(PyFrame_GetCode(frame)->co_filename); +#else + const unsigned char *file_name = PyUnicode_1BYTE_DATA(frame->f_code->co_filename); +#endif // PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + return file_name; + } + return MT_U_STRING; +} + +static const char * +get_python_function_name(PyFrameObject *frame) { + const char *func_name = NULL; + if (frame) { +#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + /* See https://docs.python.org/3.11/whatsnew/3.11.html#pyframeobject-3-11-hiding */ + func_name = (const char *) PyUnicode_1BYTE_DATA(PyFrame_GetCode(frame)->co_name); +#else + func_name = (const char *) PyUnicode_1BYTE_DATA(frame->f_code->co_name); +#endif // PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 11 + return func_name; + } + return MT_STRING; +} + +int get_python_line_number(PyFrameObject *frame) { + if (frame) { + return PyFrame_GetLineNumber(frame); + } + return 0; +} + +/** + * + * Usage: + * write_frame_data_to_outfile(stdout, PyEval_GetFrame(), PyTrace_LINE, Py_None); + * + * @param outfile + * @param frame + * @param what + * @param arg + */ +static void +write_frame_data_to_outfile(FILE *outfile, PyFrameObject *frame) { + if (frame) { + fprintf(outfile, + "%-80s %6d %-24s", + get_python_file_name(frame), + get_python_line_number(frame), + get_python_function_name(frame) + ); + } else { + fprintf(outfile, "No Python frame available."); + } +} + +static const char *watch_event_name(PyDict_WatchEvent event) { + switch (event) { + case PyDict_EVENT_ADDED: + return "PyDict_EVENT_ADDED"; + break; + case PyDict_EVENT_MODIFIED: + return "PyDict_EVENT_MODIFIED"; + break; + case PyDict_EVENT_DELETED: + return "PyDict_EVENT_DELETED"; + break; + case PyDict_EVENT_CLONED: + return "PyDict_EVENT_CLONED"; + break; + case PyDict_EVENT_CLEARED: + return "PyDict_EVENT_CLEARED"; + break; + case PyDict_EVENT_DEALLOCATED: + return "PyDict_EVENT_DEALLOCATED"; + break; + default: + Py_UNREACHABLE(); + break; + } + return "PyDict_EVENT_UNKNOWN"; +} + + +// Verbose dictionary callback function prints out Python file/line, dictionary, key and new value. +static int dict_watcher_verbose(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { + fprintf(stdout, "Dict @ 0x%p: ", (void *)dict); + write_frame_data_to_outfile(stdout, PyEval_GetFrame()); + fprintf(stdout, " Event: %-24s", watch_event_name(event)); + fprintf(stdout, "\n"); + if (dict) { + fprintf(stdout, " Dict: "); + PyObject_Print(dict, stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, " Dict: NULL"); + } + fprintf(stdout, "\n"); + if (key) { + fprintf(stdout, " Key (%s): ", Py_TYPE(key)->tp_name); + PyObject_Print(key, stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, " Key : NULL"); + } + fprintf(stdout, "\n"); + if (new_value) { + fprintf(stdout, " New value (%s): ", Py_TYPE(new_value)->tp_name); + PyObject_Print(new_value, stdout, Py_PRINT_RAW); + } else { + fprintf(stdout, " New value : NULL"); + } + fprintf(stdout, "\n"); + return 0; +} + +int dict_watcher_verbose_add(PyObject *dict) { + // Set watcher. + int watcher_id = PyDict_AddWatcher(&dict_watcher_verbose); + int api_ret_val = PyDict_Watch(watcher_id, dict); + assert(api_ret_val == 0); + return watcher_id; +} + +int dict_watcher_verbose_remove(int watcher_id, PyObject *dict) { + // Clean up. + int api_ret_val = PyDict_Unwatch(watcher_id, dict); + if (api_ret_val) { + return -1; + } + api_ret_val = PyDict_ClearWatcher(watcher_id); + if (api_ret_val) { + return -2; + } + return 0; +} + +#endif // PY_VERSION_HEX >= 0x030C0000 diff --git a/src/cpy/Watchers/DictWatcher.h b/src/cpy/Watchers/DictWatcher.h new file mode 100644 index 0000000..c4b36e8 --- /dev/null +++ b/src/cpy/Watchers/DictWatcher.h @@ -0,0 +1,40 @@ +// +// Created by Paul Ross on 30/01/2025. +// +// Explores a dict watcher: https://docs.python.org/3/c-api/dict.html#c.PyDict_AddWatcher + +#ifndef PYTHONEXTENSIONPATTERNS_DICTWATCHER_H +#define PYTHONEXTENSIONPATTERNS_DICTWATCHER_H + +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2 + * Therefore 0x030C0000 == 3.12.0 + */ +#if PY_VERSION_HEX < 0x030C0000 + +#error "Required version of Python is 3.12+ (PY_VERSION_HEX >= 0x030C0000)" + +#else + +long get_static_dict_added(void); +long get_static_dict_modified(void); +long get_static_dict_deleted(void); +long get_static_dict_cloned(void); +long get_static_dict_cleared(void); +long get_static_dict_deallocated(void); + +void dbg_PyDict_EVENT_ADDED(void); +void dbg_PyDict_EVENT_MODIFIED(void); +void dbg_PyDict_EVENT_MODIFIED_same_value_no_event(void); + + +int dict_watcher_verbose_add(PyObject *dict); + +int dict_watcher_verbose_remove(int watcher_id, PyObject *dict); + +#endif // #if PY_VERSION_HEX >= 0x030C0000 + +#endif //PYTHONEXTENSIONPATTERNS_DICTWATCHER_H diff --git a/src/cpy/Watchers/cWatchers.c b/src/cpy/Watchers/cWatchers.c new file mode 100644 index 0000000..45d8639 --- /dev/null +++ b/src/cpy/Watchers/cWatchers.c @@ -0,0 +1,170 @@ +// +// Created by Paul Ross on 31/01/2025. +// +// Provides Python accessible watchers. + +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2 + * Therefore 0x030C0000 == 3.12.0 + */ +#if PY_VERSION_HEX < 0x030C0000 + +#error "Required version of Python is 3.12+ (PY_VERSION_HEX >= 0x030C0000)" + +#else + +#pragma mark Dictionary Watcher + +#include "DictWatcher.h" + +static PyObject * +py_dict_watcher_verbose_add(PyObject *Py_UNUSED(module), PyObject *arg) { + if (!PyDict_Check(arg)) { + PyErr_Format(PyExc_TypeError, "Argument must be a dict not type %s", Py_TYPE(arg)->tp_name); + return NULL; + } + long watcher_id = dict_watcher_verbose_add(arg); + return Py_BuildValue("l", watcher_id); +} + + +static PyObject * +py_dict_watcher_verbose_remove(PyObject *Py_UNUSED(module), PyObject *args) { + long watcher_id; + PyObject *dict = NULL; + + if (!PyArg_ParseTuple(args, "lO", &watcher_id, &dict)) { + return NULL; + } + + if (!PyDict_Check(dict)) { + PyErr_Format(PyExc_TypeError, "Argument must be a dict not type %s", Py_TYPE(dict)->tp_name); + return NULL; + } + long result = dict_watcher_verbose_remove(watcher_id, dict); + return Py_BuildValue("l", result); +} + +#pragma mark Dictionary Watcher Context Manager + +typedef struct { + PyObject_HEAD + int watcher_id; + PyObject *dict; +} PyDictWatcher; + +/** Forward declaration. */ +static PyTypeObject PyDictWatcher_Type; + +#define PyDictWatcher_Check(v) (Py_TYPE(v) == &PyDictWatcher_Type) + +static PyDictWatcher * +PyDictWatcher_new(PyObject *Py_UNUSED(arg)) { + PyDictWatcher *self; + self = PyObject_New(PyDictWatcher, &PyDictWatcher_Type); + if (self == NULL) { + return NULL; + } + self->watcher_id = -1; + self->dict = NULL; + return self; +} + +static PyObject * +PyDictWatcher_init(PyDictWatcher *self, PyObject *args) { + if (!PyArg_ParseTuple(args, "O", &self->dict)) { + return NULL; + } + if (!PyDict_Check(self->dict)) { + PyErr_Format(PyExc_TypeError, "Argument must be a dictionary not a %s", Py_TYPE(self->dict)->tp_name); + return NULL; + } + Py_INCREF(self->dict); + return (PyObject *)self; +} + +static void +PyDictWatcher_dealloc(PyDictWatcher *self) { + Py_DECREF(self->dict); + PyObject_Del(self); +} + +static PyObject * +PyDictWatcher_enter(PyDictWatcher *self, PyObject *Py_UNUSED(args)) { + self->watcher_id = dict_watcher_verbose_add(self->dict); + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +PyDictWatcher_exit(PyDictWatcher *self, PyObject *Py_UNUSED(args)) { + int result = dict_watcher_verbose_remove(self->watcher_id, self->dict); + if (result) { + PyErr_Format(PyExc_RuntimeError, "dict_watcher_verbose_remove() returned %d", result); + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyMethodDef PyDictWatcher_methods[] = { + {"__enter__", (PyCFunction) PyDictWatcher_enter, METH_VARARGS, + PyDoc_STR("__enter__() -> PyDictWatcher")}, + {"__exit__", (PyCFunction) PyDictWatcher_exit, METH_VARARGS, + PyDoc_STR("__exit__(exc_type, exc_value, exc_tb) -> bool")}, + {NULL, NULL, 0, NULL} /* sentinel */ +}; + +static PyTypeObject PyDictWatcher_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "cWatchers.PyDictWatcher", + .tp_basicsize = sizeof(PyDictWatcher), + .tp_dealloc = (destructor) PyDictWatcher_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = PyDictWatcher_methods, + .tp_new = (newfunc) PyDictWatcher_new, + .tp_init = (initproc) PyDictWatcher_init +}; + +static PyMethodDef module_methods[] = { + {"py_dict_watcher_verbose_add", + (PyCFunction) py_dict_watcher_verbose_add, + METH_O, + "Adds watcher to a dictionary. Returns the watcher ID." + }, + {"py_dict_watcher_verbose_remove", + (PyCFunction) py_dict_watcher_verbose_remove, + METH_VARARGS, + "Removes the watcher ID from the dictionary." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cWatchers = { + PyModuleDef_HEAD_INIT, + .m_name = "cWatchers", + .m_doc = "Dictionary and type watchers.", + .m_size = -1, + .m_methods = module_methods, +}; + +PyMODINIT_FUNC PyInit_cWatchers(void) { + PyObject *m = PyModule_Create(&cWatchers); + if (!m) { + goto fail; + } + if (PyType_Ready(&PyDictWatcher_Type) < 0) { + goto fail; + } + if (PyModule_AddObject(m, "PyDictWatcher", (PyObject *) &PyDictWatcher_Type)) { + goto fail; + } + return m; +fail: + Py_XDECREF(m); + return NULL; +} + +#endif // #if PY_VERSION_HEX >= 0x030C0000 diff --git a/src/cpy/Watchers/watcher_example.py b/src/cpy/Watchers/watcher_example.py new file mode 100644 index 0000000..b3a251c --- /dev/null +++ b/src/cpy/Watchers/watcher_example.py @@ -0,0 +1,132 @@ +"""Example of using watchers.""" +import sys + +from cPyExtPatt import cWatchers + + +def dict_watcher_demo() -> None: + print('dict_watcher_demo():') + d = {} + with cWatchers.PyDictWatcher(d): + dd = {'age': 17, } + d.update(dd) + d['age'] = 42 + del d['age'] + d['name'] = 'Python' + d.clear() + del d + + +def dict_watcher_demo_refcount() -> None: + """Checks that the reference count of the dictionary is managed correctly by the context manager.""" + print('dict_watcher_demo_refcount():') + d = {} + print(f'Ref count pre {sys.getrefcount(d)}') + ref_count = sys.getrefcount(d) + # assert ref_count == 1 + with cWatchers.PyDictWatcher(d): + d['age'] = 42 + print(f'Ref count post {sys.getrefcount(d)}') + assert sys.getrefcount(d) == ref_count + + +def dict_watcher_add() -> None: + print('dict_watcher_add():') + d = {} + with cWatchers.PyDictWatcher(d): + d['age'] = 42 + + +def dict_watcher_add_and_replace() -> None: + print('dict_watcher_add_and_replace():') + d = {} + d['age'] = 42 + with cWatchers.PyDictWatcher(d): + d['age'] = 43 + + +def dict_watcher_add_and_del() -> None: + print('dict_watcher_add_and_del():') + d = {} + d['age'] = 42 + with cWatchers.PyDictWatcher(d): + del d['age'] + + +def dict_watcher_add_and_clear() -> None: + print('dict_watcher_add_and_clear():') + d = {} + d['age'] = 42 + with cWatchers.PyDictWatcher(d): + d.clear() + + +def dict_watcher_del() -> None: + print('dict_watcher_del():') + d = {} + d['age'] = 42 + with cWatchers.PyDictWatcher(d): + del d + + +def dict_watcher_cloned() -> None: + print('dict_watcher_cloned():') + d = {} + with cWatchers.PyDictWatcher(d): + dd = {'age': 42, } + d.update(dd) + + +def dict_watcher_deallocated() -> None: + print('dict_watcher_deallocated():') + d = {'age': 42, } + dd = d + with cWatchers.PyDictWatcher(dd): + del d + del dd + + +def dict_watcher_add_no_context_manager() -> None: + print('dict_watcher_add_no_context_manager():') + d = {} + watcher_id = cWatchers.py_dict_watcher_verbose_add(d) + d['age'] = 42 + cWatchers.py_dict_watcher_verbose_remove(watcher_id, d) + + +# def temp() -> None: +# d = {} +# cm = cWatchers.PyDictWatcher(d) +# cmm = cm.__enter__(d) +# d['age'] = 42 +# d['age'] = 43 +# cmm.__exit__() +# +# +# def temp_2() -> None: +# d = {} +# watcher_id = cWatchers.py_dict_watcher_verbose_add(d) +# d['age'] = 22 +# d['age'] = 23 +# del d['age'] +# cWatchers.py_dict_watcher_verbose_remove(watcher_id, d) + + +def main() -> int: + # temp() + # temp_2() + dict_watcher_demo() + dict_watcher_demo_refcount() + dict_watcher_add() + dict_watcher_add_and_replace() + dict_watcher_add_and_del() + dict_watcher_add_and_clear() + dict_watcher_del() + dict_watcher_cloned() + dict_watcher_deallocated() + dict_watcher_add_no_context_manager() + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/src/cpy/cpp/cUnicode.cpp b/src/cpy/cpp/cUnicode.cpp new file mode 100644 index 0000000..6498a3c --- /dev/null +++ b/src/cpy/cpp/cUnicode.cpp @@ -0,0 +1,221 @@ +// +// cUnicode.cpp +// PyCppUnicode +// +// Created by Paul Ross on 16/04/2018. +// Copyright (c) 2018-2024 Paul Ross. All rights reserved. +// + +#include + +#include +#include +#include +#include + +/** Converting Python bytes and Unicode to and from std::string + * Convert a PyObject to a std::string and return 0 if successful. + * If py_str is Unicode than treat it as UTF-8. + * This works with Python 2.7 and Python 3.4 onwards. + */ +static int +py_object_to_std_string(const PyObject *py_object, std::string &result, bool utf8_only = true) { + result.clear(); + if (PyBytes_Check(py_object)) { + result = std::string(PyBytes_AS_STRING(py_object)); + return 0; + } + if (PyByteArray_Check(py_object)) { + result = std::string(PyByteArray_AS_STRING(py_object)); + return 0; + } + // Must be unicode then. + if (!PyUnicode_Check(py_object)) { + PyErr_Format(PyExc_ValueError, + "In %s \"py_str\" failed PyUnicode_Check()", + __FUNCTION__); + return -1; + } + if (PyUnicode_READY(py_object)) { + PyErr_Format(PyExc_ValueError, + "In %s \"py_str\" failed PyUnicode_READY()", + __FUNCTION__); + return -2; + } + if (utf8_only && PyUnicode_KIND(py_object) != PyUnicode_1BYTE_KIND) { + PyErr_Format(PyExc_ValueError, + "In %s \"py_str\" not utf-8", + __FUNCTION__); + return -3; + } + result = std::string((char *) PyUnicode_1BYTE_DATA(py_object)); + return 0; +} + +static PyObject * +std_string_to_py_bytes(const std::string &str) { + return PyBytes_FromStringAndSize(str.c_str(), str.size()); +} + +static PyObject * +std_string_to_py_bytearray(const std::string &str) { + return PyByteArray_FromStringAndSize(str.c_str(), str.size()); +} + +static PyObject * +std_string_to_py_utf8(const std::string &str) { + // Equivelent to: + // PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, str.c_str(), str.size()); + return PyUnicode_FromStringAndSize(str.c_str(), str.size()); +} + +static PyObject * +py_object_to_string_and_back(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *py_object = NULL; + + if (!PyArg_ParseTuple(args, "O", &py_object)) { + return NULL; + } + std::string str_result; + int err_code = py_object_to_std_string(py_object, str_result, false); + if (err_code) { + PyErr_Format(PyExc_ValueError, + "In %s \"py_object_to_std_string\" failed with error code %d", + __FUNCTION__, + err_code + ); + return NULL; + } + if (PyBytes_Check(py_object)) { + return std_string_to_py_bytes(str_result); + } + if (PyByteArray_Check(py_object)) { + return std_string_to_py_bytearray(str_result); + } + if (PyUnicode_Check(py_object)) { + return std_string_to_py_utf8(str_result); + } + PyErr_Format(PyExc_ValueError, + "In %s does not support python type %s", + __FUNCTION__, + Py_TYPE(py_object)->tp_name + ); + return NULL; +} + +template +static void dump_string(const std::basic_string &str) { + std::cout << "String size: " << str.size(); + std::cout << " word size: " << sizeof(T) << std::endl; + for (size_t i = 0; i < str.size(); ++i) { + std::cout << "0x" << std::hex << std::setfill('0'); + std::cout << std::setw(8) << static_cast(str[i]); + std::cout << std::setfill(' '); + std::cout << " " << std::dec << std::setw(8) << static_cast(str[i]); + std::cout << " \"" << str[i] << "\"" << std::endl; + } +} + +static PyObject * +unicode_1_to_string_and_back(PyObject *py_str) { + assert(PyUnicode_KIND(py_str) == PyUnicode_1BYTE_KIND); + std::string result = std::string((char *) PyUnicode_1BYTE_DATA(py_str)); + dump_string(result); + return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, + result.c_str(), + result.size()); +} + +static PyObject * +unicode_2_to_string_and_back(PyObject *py_str) { + assert(PyUnicode_KIND(py_str) == PyUnicode_2BYTE_KIND); + // NOTE: std::u16string is a std::basic_string + std::u16string result = std::u16string((char16_t *) PyUnicode_2BYTE_DATA(py_str)); + dump_string(result); + return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, + result.c_str(), + result.size()); +} + +static PyObject * +unicode_4_to_string_and_back(PyObject *py_str) { + assert(PyUnicode_KIND(py_str) == PyUnicode_4BYTE_KIND); + // NOTE: std::u32string is a std::basic_string + std::u32string result = std::u32string((char32_t *) PyUnicode_4BYTE_DATA(py_str)); + dump_string(result); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + result.c_str(), + result.size()); +} + +static void +unicode_dump_as_1byte_string(PyObject *py_str) { + Py_ssize_t len = PyUnicode_GET_LENGTH(py_str) * PyUnicode_KIND(py_str); + std::string result = std::string((char *) PyUnicode_1BYTE_DATA(py_str), len); + std::cout << "unicode_dump_as_1byte_string();" << std::endl; + dump_string(result); +} + +static PyObject * +unicode_to_string_and_back(PyObject *Py_UNUSED(module), PyObject *args) { + PyObject *py_str = NULL; + PyObject *ret_val = NULL; + if (! PyArg_ParseTuple(args, "U", &py_str)) { + return NULL; + } + unicode_dump_as_1byte_string(py_str); + std::cout << "Native:" << std::endl; + switch (PyUnicode_KIND(py_str)) { + case PyUnicode_1BYTE_KIND: + ret_val = unicode_1_to_string_and_back(py_str); + break; + case PyUnicode_2BYTE_KIND: + ret_val = unicode_2_to_string_and_back(py_str); + break; + case PyUnicode_4BYTE_KIND: + ret_val = unicode_4_to_string_and_back(py_str); + break; + default: + PyErr_Format(PyExc_ValueError, + "In %s argument is not recognised as a Unicode 1, 2, 4 byte string", + __FUNCTION__); + ret_val = NULL; + break; + } + return ret_val; +} + +static PyMethodDef cUnicode_Methods[] = { + { + "unicode_to_string_and_back", + (PyCFunction) unicode_to_string_and_back, + METH_VARARGS, + "Convert a Python unicode string to std::string and back." + }, + { + "py_object_to_string_and_back", + (PyCFunction) py_object_to_string_and_back, + METH_VARARGS, + "Convert a Python unicode string, bytes, bytearray to std::string and back." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyModuleDef cUnicodemodule = { + PyModuleDef_HEAD_INIT, + "cUnicode", + "cUnicode works with unicode strings.", + -1, + cUnicode_Methods, + NULL, NULL, NULL, NULL +}; + +PyMODINIT_FUNC +PyInit_cUnicode(void) { + PyObject *m; + + m = PyModule_Create(&cUnicodemodule); + if (m == NULL) + return NULL; + return m; +} diff --git a/src/cpy/cpp/placement_new.cpp b/src/cpy/cpp/placement_new.cpp new file mode 100644 index 0000000..7d5c3d4 --- /dev/null +++ b/src/cpy/cpp/placement_new.cpp @@ -0,0 +1,187 @@ +// +// Created by Paul Ross on 06/09/2022. +// + +#define PY_SSIZE_T_CLEAN + +#include +#include "structmember.h" + +#include +#include + +/** + * A simple class that contains a string but reports its method calls. + */ +class Verbose { +public: + Verbose() : Verbose("Default") { + std::cout << "Default constructor at " << std::hex << (void *) this << std::dec; + std::cout << " with argument \"" << m_str << "\"" << std::endl; + } + + /// Constructor reserves 256MB to illustrate memory usage visible in the process RSS. + explicit Verbose(const std::string &str) : m_str(str), m_buffer(1024 * 1024 * 256, ' ') { + std::cout << "Constructor at " << std::hex << (void *) this << std::dec; + std::cout << " with argument \"" << m_str << "\"" << " buffer len: " << m_buffer.size() << std::endl; + } + + Verbose &operator=(const Verbose &rhs) { + std::cout << "operator= at " << std::hex << (void *) this << std::dec; + std::cout << " m_str: \"" << m_str << "\""; + std::cout << " rhs at " << std::hex << (void *) &rhs << std::dec; + std::cout << " rhs.m_str: \"" << rhs.m_str << "\"" << std::endl; + if (this != &rhs) { + m_str = rhs.m_str; + } + return *this; + } + + void print(const char *message = NULL) { + if (message) { + std::cout << message << ": Verbose object at " << std::hex << (void *) this << std::dec; + std::cout << " m_str: \"" << m_str << "\"" << std::endl; + } else { + std::cout << " Verbose object at " << std::hex << (void *) this << std::dec; + std::cout << " m_str: \"" << m_str << "\"" << std::endl; + } + } + + [[nodiscard]] ssize_t buffer_size() const { + return sizeof(Verbose) + 2 * sizeof(std::string) + m_str.size() + m_buffer.size(); + } + + ~Verbose() { + std::cout << "Destructor at " << std::hex << (void *) this << std::dec; + std::cout << " m_str: \"" << m_str << "\"" << std::endl; + } + +private: + std::string m_str; + // m_buffer is just a large string to provoke the memory manager and detect leaks. + std::string m_buffer; +}; + +typedef struct { + PyObject_HEAD + Verbose Attr; + Verbose *pAttr; +} CppCtorDtorInPyObject; + +static PyObject * +CppCtorDtorInPyObject_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { + printf("-- %s()\n", __FUNCTION__); + CppCtorDtorInPyObject *self; + self = (CppCtorDtorInPyObject *) type->tp_alloc(type, 0); + if (self != NULL) { + // Placement new used for direct allocation. + new(&self->Attr) Verbose; + self->Attr.print("Initial self->Attr"); + // Dynamically allocated new. + self->pAttr = new Verbose("pAttr"); + if (self->pAttr == NULL) { + Py_DECREF(self); + return NULL; + } else { + self->pAttr->print("Initial self->pAttr"); + } + } + return (PyObject *) self; +} + +//static int +//CppCtorDtorInPyObject_init(CppCtorDtorInPyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args), +// PyObject *Py_UNUSED(kwds)) { +// printf("-- %s()\n", __FUNCTION__); +// return 0; +//} + +static void +CppCtorDtorInPyObject_dealloc(CppCtorDtorInPyObject *self) { + printf("-- %s()\n", __FUNCTION__); + self->Attr.print("self->Attr before delete"); + // For self->Attr call the destructor directly. + self->Attr.~Verbose(); +// delete (&self->Attr);// self->Attr; +// ::operator delete (&self->Attr);// self->Attr; + self->pAttr->print("self->pAttr before delete"); + // For self->pAttr use delete. + delete self->pAttr; + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +CppCtorDtorInPyObject_print(CppCtorDtorInPyObject *self, PyObject *Py_UNUSED(ignored)) { + printf("-- %s()\n", __FUNCTION__); + self->Attr.print("self->Attr"); + self->pAttr->print("self->pAttr"); + Py_RETURN_NONE; +} + +/// Best guess of the size of the Verbose object(s). +static PyObject * +CppCtorDtorInPyObject_buffer_size(CppCtorDtorInPyObject *self, PyObject *Py_UNUSED(ignored)) { + printf("-- %s()\n", __FUNCTION__); + Py_ssize_t ret = 0; + ret += self->Attr.buffer_size(); + ret += self->pAttr->buffer_size(); + return Py_BuildValue("n", ret); +} + +static PyMethodDef CppCtorDtorInPyObject_methods[] = { + { + "print", + (PyCFunction) CppCtorDtorInPyObject_print, + METH_NOARGS, + "Print the contents of the object." + }, + { + "buffer_size", + (PyCFunction) CppCtorDtorInPyObject_buffer_size, + METH_NOARGS, + "The memory usage of the object." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static PyTypeObject CppCtorDtorInPyObjectType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "CppCtorDtorInPyObject", + .tp_basicsize = sizeof(CppCtorDtorInPyObject), + .tp_itemsize = 0, + .tp_dealloc = (destructor) CppCtorDtorInPyObject_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = "CppCtorDtorInPyObject object", + .tp_methods = CppCtorDtorInPyObject_methods, +// .tp_init = (initproc) CppCtorDtorInPyObject_init, + .tp_new = CppCtorDtorInPyObject_new, +}; + +static PyModuleDef placement_new_module = { + PyModuleDef_HEAD_INIT, + .m_name = "placement_new", + .m_doc = "Example module that creates an C++ extension type containing custom objects.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_placement_new(void) { +// printf("-- %s()\n", __FUNCTION__); + PyObject * m = PyModule_Create(&placement_new_module); + if (m == NULL) { + return NULL; + } + + if (PyType_Ready(&CppCtorDtorInPyObjectType) < 0) { + Py_DECREF(m); + return NULL; + } + + Py_INCREF(&CppCtorDtorInPyObjectType); + if (PyModule_AddObject(m, "CppCtorDtorInPyObject", (PyObject *) &CppCtorDtorInPyObjectType) < 0) { + Py_DECREF(&CppCtorDtorInPyObjectType); + Py_DECREF(m); + return NULL; + } + return m; +} diff --git a/src/cpy/pyextpatt_util.c b/src/cpy/pyextpatt_util.c new file mode 100644 index 0000000..2a75e34 --- /dev/null +++ b/src/cpy/pyextpatt_util.c @@ -0,0 +1,25 @@ +// +// Created by Paul Ross on 30/01/2025. +// + +#include "pyextpatt_util.h" + +/* This is used to guarantee that Python is not caching a string value when we want to check the + * reference counts after each string creation. + * */ +static long debug_test_count = 0L; + +PyObject * +new_unique_string(const char *function_name, const char *suffix) { + PyObject *value = NULL; + if (suffix) { + value = PyUnicode_FromFormat("%s-%s-%ld", function_name, suffix, debug_test_count); + } else { + value = PyUnicode_FromFormat("%s-%ld", function_name, debug_test_count); + } + /* To view in the debugger. */ + Py_UCS1 *buffer = PyUnicode_1BYTE_DATA(value); + assert(buffer); + ++debug_test_count; + return value; +} diff --git a/src/cpy/pyextpatt_util.h b/src/cpy/pyextpatt_util.h new file mode 100644 index 0000000..3b6cb36 --- /dev/null +++ b/src/cpy/pyextpatt_util.h @@ -0,0 +1,14 @@ +// +// Created by Paul Ross on 30/01/2025. +// + +#ifndef PYTHONEXTENSIONPATTERNS_PYEXTPATT_UTIL_H +#define PYTHONEXTENSIONPATTERNS_PYEXTPATT_UTIL_H + +#define PPY_SSIZE_T_CLEAN + +#include "Python.h" + +PyObject *new_unique_string(const char *function_name, const char *suffix); + +#endif //PYTHONEXTENSIONPATTERNS_PYEXTPATT_UTIL_H diff --git a/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate b/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate index 7256149..e3a6550 100644 Binary files a/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate and b/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/project.xcworkspace/xcuserdata/paulross.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/xcuserdata/paulross.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist b/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/xcuserdata/paulross.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist index e81d915..fdc1005 100644 --- a/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/xcuserdata/paulross.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist +++ b/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList.xcodeproj/xcuserdata/paulross.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist @@ -24,6 +24,7 @@ continueAfterRunningActions = "No" symbolName = "ScList_append" moduleName = "PythonSubclassList" + usesParentBreakpointCondition = "Yes" urlString = "file:///Users/paulross/Documents/workspace/PythonExtensionPatterns/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/SubclassList.c" timestampString = "486418359.147248" startingColumnNumber = "9223372036854775807" @@ -38,6 +39,7 @@ continueAfterRunningActions = "No" symbolName = "ScList_append" moduleName = "ScList.so" + usesParentBreakpointCondition = "Yes" urlString = "file:///Users/paulross/Documents/workspace/PythonExtensionPatterns/src/debugging/XcodeExample/PythonSubclassList/PythonSubclassList/SubclassList.c" timestampString = "486418359.297105" startingColumnNumber = "9223372036854775807" @@ -61,8 +63,8 @@ endingColumnNumber = "9223372036854775807" startingLineNumber = "49" endingLineNumber = "49" - landmarkName = "call_super_pyname()" - landmarkType = "7"> + landmarkName = "call_super_pyname" + landmarkType = "9"> + offsetFromSymbolStart = "405"> + +#include + +#include "DebugContainers.h" +#include "DictWatcher.h" + +/** + * Get the current working directory using \c getcwd(). + * + * @return The current working directory or NULL on failure. + */ +const char *current_working_directory(const char *extend) { + static char cwd[4096]; + if (getcwd(cwd, sizeof(cwd)) == NULL) { + fprintf(stderr, "%s(): Can not get current working directory.\n", __FUNCTION__); + return NULL; + } + if (extend) { + if (snprintf(cwd + strlen(cwd), strlen(extend) + 2, "/%s", extend) == 0) { + fprintf(stderr, "%s(): Can not compose buffer.\n", __FUNCTION__); + return NULL; + } + } + return cwd; +} + +#if 0 +/** Takes a path and adds it to sys.paths by calling PyRun_SimpleString. + * This does rather laborious C string concatenation so that it will work in + * a primitive C environment. + * + * Returns 0 on success, non-zero on failure. + */ +int add_path_to_sys_module(const char *path) { + int ret = 0; + const char *prefix = "import sys\nsys.path.append(\""; + const char *suffix = "\")\n"; + char *command = (char*)malloc(strlen(prefix) + + strlen(path) + + strlen(suffix) + + 1); + if (! command) { + return -1; + } + strcpy(command, prefix); + strcat(command, path); + strcat(command, suffix); + ret = PyRun_SimpleString(command); +#ifdef DEBUG + printf("Calling PyRun_SimpleString() with:\n"); + printf("%s", command); + printf("PyRun_SimpleString() returned: %d\n", ret); + fflush(stdout); +#endif + free(command); + return ret; +} + +int import_call_execute_no_args(const char *module_name, const char *function_name) { + int return_value = 0; + PyObject *pModule = NULL; + PyObject *pFunc = NULL; + PyObject *pResult = NULL; + + assert(! PyErr_Occurred()); + pModule = PyImport_ImportModule(module_name); + if (! pModule) { + fprintf(stderr, "%s: Failed to load module \"%s\"\n", __FUNCTION__ , module_name); + return_value = -3; + goto except; + } + assert(! PyErr_Occurred()); + pFunc = PyObject_GetAttrString(pModule, function_name); + if (! pFunc) { + fprintf(stderr, + "%s: Can not find function \"%s\"\n", __FUNCTION__, function_name); + return_value = -4; + goto except; + } + assert(! PyErr_Occurred()); + if (! PyCallable_Check(pFunc)) { + fprintf(stderr, + "%s: Function \"%s\" is not callable\n", __FUNCTION__, function_name); + return_value = -5; + goto except; + } + assert(! PyErr_Occurred()); + pResult = PyObject_CallObject(pFunc, NULL); + if (! pResult) { + fprintf(stderr, "%s: Function call %s failed\n", __FUNCTION__, function_name); + return_value = -6; + goto except; + } + assert(! PyErr_Occurred()); +goto finally; + except: + assert(PyErr_Occurred()); + PyErr_Print(); +finally: + Py_XDECREF(pFunc); + Py_XDECREF(pModule); + Py_XDECREF(pResult); + Py_Finalize(); + return return_value; +} + +#endif + +#pragma mark - Tuples +void dbg_PyTuple(void) { + dbg_PyTuple_SetItem_steals(); + dbg_PyTuple_SET_ITEM_steals(); + dbg_PyTuple_SetItem_steals_replace(); + dbg_PyTuple_SET_ITEM_steals_replace(); + dbg_PyTuple_SetItem_replace_with_same(); + dbg_PyTuple_SET_ITEM_replace_with_same(); + dbg_PyTuple_SetIem_NULL(); + dbg_PyTuple_SET_ITEM_NULL(); + dbg_PyTuple_SetIem_NULL_SetItem(); + dbg_PyTuple_SET_ITEM_NULL_SET_ITEM(); + dbg_PyTuple_SetItem_fails_not_a_tuple(); + dbg_PyTuple_SetItem_fails_out_of_range(); + dbg_PyTuple_PyTuple_Pack(); + dbg_PyTuple_Py_BuildValue(); +#if ACCEPT_SIGSEGV + /* Comment out as desired. */ + dbg_PyTuple_SetItem_SIGSEGV_on_same_value(); +#endif +} + +#pragma mark - Lists +void dbg_PyList(void) { + dbg_PyList_SetItem_steals(); + dbg_PyList_SET_ITEM_steals(); + dbg_PyList_SetItem_steals_replace(); + dbg_PyList_SET_ITEM_steals_replace(); + dbg_PyList_SetItem_replace_with_same(); + dbg_PyList_SET_ITEM_replace_with_same(); + dbg_PyList_SetIem_NULL(); + dbg_PyList_SET_ITEM_NULL(); + dbg_PyList_SetIem_NULL_SetItem(); + dbg_PyList_SET_ITEM_NULL_SET_ITEM(); + dbg_PyList_SetItem_fails_not_a_tuple(); + dbg_PyList_SetItem_fails_out_of_range(); + dbg_PyList_Append(); + dbg_PyList_Append_fails_not_a_list(); + dbg_PyList_Append_fails_NULL(); + dbg_PyList_Insert(); + dbg_PyList_Insert_Is_Truncated(); + dbg_PyList_Insert_Negative_Index(); + dbg_PyList_Insert_fails_not_a_list(); + dbg_PyList_Insert_fails_NULL(); + dbg_PyList_Py_BuildValue(); +#if ACCEPT_SIGSEGV + /* Comment out as desired. */ + dbg_PyList_SetItem_SIGSEGV_on_same_value(); +#endif +} + +#pragma mark - Dictionaries +void dbg_PyDict(void) { + dbg_PyDict_SetItem_increments(); + +#if ACCEPT_SIGSEGV + dbg_PyDict_SetItem_NULL_key(); + dbg_PyDict_SetItem_NULL_value(); +#endif // ACCEPT_SIGSEGV + + dbg_PyDict_SetItem_fails_not_a_dict(); + dbg_PyDict_SetItem_fails_not_hashable(); + dbg_PyDict_SetDefault_default_unused(); + dbg_PyDict_SetDefault_default_used(); + dbg_PyDict_SetDefaultRef_default_unused(); + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + dbg_PyDict_SetDefaultRef_default_used(); + dbg_PyDict_SetDefaultRef_default_unused_result_non_null(); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + + dbg_PyDict_GetItem(); + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + dbg_PyDict_GetItemRef(); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + + dbg_PyDict_GetItemWithError_fails(); + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + dbg_PyDict_Pop_key_present(); + dbg_PyDict_Pop_key_absent(); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 13 + +#if ACCEPT_SIGSEGV + /* Comment out as desired. */ + dbg_PyDict_SetItem_SIGSEGV_on_key_NULL(); + dbg_PyDict_SetItem_SIGSEGV_on_value_NULL(); + dbg_PyDict_GetItem_key_NULL(); +#endif + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 12 + // Watchers + dbg_PyDict_EVENT_ADDED(); + dbg_PyDict_EVENT_MODIFIED(); + dbg_PyDict_EVENT_MODIFIED_same_value_no_event(); +#endif // #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 12 +} + +#pragma mark - Sets +void dbg_PySet(void) { + dbg_PySet_Add(); + dbg_PySet_Discard(); + dbg_PySet_Pop(); +} + +#pragma mark - Struct Sequence +void dbg_PyStructSequence(void) { + dbg_PyStructSequence_simple_ctor(); + dbg_PyStructSequence_setitem_abandons(); + dbg_PyStructSequence_n_in_sequence_too_large(); + dbg_PyStructSequence_with_unnamed_field(); +} + +int main(int argc, const char * argv[]) { + // insert code here... + printf("Hello, World!\n"); + Py_Initialize(); + const char *cwd = current_working_directory(".."); + int failure = 0; + + int32_t py_version_hex = PY_VERSION_HEX; + printf("Python version %d.%d.%d Release level: 0x%x Serial: %d Numeric: %12d 0x%08x\n", + PY_MAJOR_VERSION, PY_MINOR_VERSION, PY_MICRO_VERSION, + PY_RELEASE_LEVEL, PY_RELEASE_SERIAL, + py_version_hex, py_version_hex + ); +// failure = add_path_to_sys_module(cwd); +// if (failure) { +// printf("add_path_to_sys_module(): Failed with error code %d\n", failure); +// goto finally; +// } + +// failure = dbg_cPyRefs(); +// if (failure) { +// printf("dbg_cPyRefs(): Failed with error code %d\n", failure); +// return -1; +// } + +// /* cPyExtPatt.cRefCount tests*/ +// failure = test_dbg_cRefCount(); +// if (failure) { +// printf("test_dbg_cRefCount(): Failed with error code %d\n", failure); +// goto finally; +// } + + dbg_PyTuple(); + dbg_PyList(); + dbg_PyDict(); + dbg_PySet(); + dbg_PyStructSequence(); + + printf("Ran all tests, failure=%d\n", failure); + printf("Bye, bye!\n"); + printf("\n"); + return failure; +} + diff --git a/src/memleak.py b/src/memleak.py index fb8892d..744274f 100644 --- a/src/memleak.py +++ b/src/memleak.py @@ -1,9 +1,11 @@ -''' +""" Created on 26 May 2015 @author: paulross -''' -import cPyRefs + +Deliberately create a 1GB memory leak. +""" +from cPyExtPatt import cPyRefs s = ' ' * 1024**3 cPyRefs.incref(s) diff --git a/src/minimal.py b/src/minimal.py deleted file mode 100644 index ad35e5a..0000000 --- a/src/minimal.py +++ /dev/null @@ -1 +0,0 @@ -print("Hello World") diff --git a/src/pidmon.py b/src/pidmon.py index c01fd28..7890b33 100644 --- a/src/pidmon.py +++ b/src/pidmon.py @@ -2,14 +2,18 @@ Created on 5 May 2015 @author: paulross + +A simple memory monitoring tool. """ import sys import time import psutil -def memMon(pid, freq=1.0): - proc = psutil.Process(pid) + +def memory_monitor(process_id: int, frequency: float = 1.0) -> None: + """Print out memory usage of a process at regular intervals.""" + proc = psutil.Process(process_id) print(proc.memory_info_ex()) prev_mem = None while True: @@ -20,7 +24,7 @@ def memMon(pid, freq=1.0): else: print('{:10.3f} [Mb] {:+10.3f} [Mb]'.format(mem, mem - prev_mem)) prev_mem = mem - time.sleep(freq) + time.sleep(frequency) except KeyboardInterrupt: try: input(' Pausing memMon, to continue, Ctrl-C to end...') @@ -28,10 +32,15 @@ def memMon(pid, freq=1.0): print('\n') return -if __name__ == '__main__': + +def main() -> int: if len(sys.argv) < 2: print('Usage: python pidmon.py ') - sys.exit(1) + return -1 pid = int(sys.argv[1]) - memMon(pid) - sys.exit(0) + memory_monitor(pid) + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/src/scratch.c b/src/scratch.c index 76099fe..8496151 100644 --- a/src/scratch.c +++ b/src/scratch.c @@ -5,6 +5,8 @@ // Created by Paul Ross on 04/04/2015. // Copyright (c) 2015 Paul Ross. All rights reserved. // +// Just a scratch area for formatting code that goes into the documentation. +// This is not code that is built. #include "Python.h" @@ -12,14 +14,15 @@ #include -void leak() { +void leak(void) { char *p; p = malloc(1024); + fprintf(stdout, "malloc(1024) returns %s", p); } -void access_after_free() { +void access_after_free(void) { char *p; p = malloc(1024); @@ -32,7 +35,7 @@ void access_after_free() { #include "Python.h" -void py_leak() { +void py_leak(void) { PyObject *pObj = NULL; /* Object creation, ref count = 1. */ @@ -43,7 +46,7 @@ void py_leak() { #include "Python.h" -void py_access_after_free() { +void py_access_after_free(void) { PyObject *pObj = NULL; /* Object creation, ref count = 1. */ @@ -78,7 +81,7 @@ PyObject *bad_incref(PyObject *pObj) { } -void bad_steal() { +void bad_steal(void) { PyObject *v, *r; @@ -105,7 +108,7 @@ static PyObject *pop_and_print_BAD(PyObject *pList) { pLast = PyList_GetItem(pList, PyList_Size(pList) - 1); fprintf(stdout, "Ref count was: %zd\n", pLast->ob_refcnt); - do_something(pList); /* Dragons ahoy me hearties! */ + //do_something(pList); /* Dragons ahoy me hearties! */ fprintf(stdout, "Ref count now: %zd\n", pLast->ob_refcnt); PyObject_Print(pLast, stdout, 0); fprintf(stdout, "\n"); diff --git a/src/setup.py b/src/setup.py deleted file mode 100644 index b062112..0000000 --- a/src/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -""" Usage: -python3 setup.py build - -Created on May 30, 2013 - -@author: paulross -""" -import os - -DEBUG = True - -extra_compile_args=["-std=c99", ] -if DEBUG: - extra_compile_args += ["-g3", "-O0", "-DDEBUG=1",] -else: - extra_compile_args += ["-DNDEBUG", "-Os"] - - -from distutils.core import setup, Extension -setup( - name = 'cPyExtPatt', - version = '0.1.0', - author = 'Paul Ross', - author_email = 'cpipdev@gmail.com', - maintainer = 'Paul Ross', - maintainer_email = 'cpipdev@gmail.com', - description = 'Python Extension Patterns.', - long_description = """Examples of good and bad practice with Python Extensions.""", - platforms = ['Mac OSX', 'POSIX',], - classifiers = [ - 'Development Status :: 3 - Alpha', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: C', - 'Programming Language :: Python', - 'Topic :: Programming', - ], - license = 'GNU General Public License v2 (GPLv2)', - ext_modules=[ - Extension("cExceptions", sources=['cExceptions.c',], - include_dirs = ['/usr/local/include',], # os.path.join(os.getcwd(), 'include'),], - library_dirs = [os.getcwd(),], # path to .a or .so file(s) - extra_compile_args=extra_compile_args, - ), - Extension("cModuleGlobals", sources=['cModuleGlobals.c',], - include_dirs = ['/usr/local/include',], # os.path.join(os.getcwd(), 'include'),], - library_dirs = [os.getcwd(),], # path to .a or .so file(s) - extra_compile_args=extra_compile_args, - ), - Extension("cObj", sources=['cObjmodule.c',], - include_dirs = ['/usr/local/include',], # os.path.join(os.getcwd(), 'include'),], - library_dirs = [os.getcwd(),], # path to .a or .so file(s) - extra_compile_args=extra_compile_args, - ), - Extension("cParseArgs", sources=['cParseArgs.c',], - include_dirs = ['/usr/local/include',], # os.path.join(os.getcwd(), 'include'),], - library_dirs = [os.getcwd(),], # path to .a or .so file(s) - extra_compile_args=extra_compile_args, - ), - Extension("cPyRefs", sources=['cPyRefs.c',], - include_dirs = ['/usr/local/include',], # os.path.join(os.getcwd(), 'include'),], - library_dirs = [os.getcwd(),], # path to .a or .so file(s) - #libraries = ['jpeg',], - extra_compile_args=extra_compile_args, - ), - ] -) diff --git a/tests/unit/test_c_capsules.py b/tests/unit/test_c_capsules.py new file mode 100644 index 0000000..c5c3c5a --- /dev/null +++ b/tests/unit/test_c_capsules.py @@ -0,0 +1,271 @@ +import datetime +import sys +import zoneinfo + +import pytest + +from cPyExtPatt.Capsules import spam +from cPyExtPatt.Capsules import spam_capsule +from cPyExtPatt.Capsules import spam_client +from cPyExtPatt.Capsules import datetimetz + + +def test_spam(): + result = spam.system("ls -l") + assert result == 0 + + +def test_spam_capsule(): + result = spam_capsule.system("ls -l") + assert result == 0 + + +@pytest.mark.skipif(not (sys.version_info.minor <= 10), reason='Python 3.9, 3.10') +def test_spam_capsule__C_API_39_310(): + print() + print(spam_capsule._C_API) + assert str(spam_capsule._C_API).startswith(', )" + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_spam_capsule__C_API_311_plus(): + print() + print(spam_capsule._C_API) + assert str(spam_capsule._C_API).startswith(', )" + + +def test_spam_client(): + result = spam_client.system("ls -l") + assert result == 0 + + +def test_datetimetz_datetimetz_mro(): + mro = datetimetz.datetimetz.__mro__ + assert [str(v) for v in mro] == [ + "", + "", + "", + "", + ] + + +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + (2024, 7, 15, 10, 21, 14), + {'tzinfo': zoneinfo.ZoneInfo('Europe/London')}, + '2024-07-15 10:21:14+01:00', + ), + ) +) +def test_datetimetz_datetimetz_str(args, kwargs, expected): + d = datetimetz.datetimetz(*args, **kwargs) + assert str(d) == expected + + +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + (2024, 7, 15, 10, 21, 14), + {'tzinfo': zoneinfo.ZoneInfo('Europe/London')}, + ( + "datetimetz.datetimetz(2024, 7, 15, 10, 21, 14," + " tzinfo=zoneinfo.ZoneInfo(key='Europe/London'))" + ), + ), + ) +) +def test_datetimetz_datetimetz_repr(args, kwargs, expected): + d = datetimetz.datetimetz(*args, **kwargs) + assert repr(d) == expected + + +def test_datetimetz_datetimetz_get_attributes(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + assert d.year == 2024 + assert d.month == 7 + assert d.day == 15 + assert d.hour == 10 + assert d.minute == 21 + assert d.second == 14 + assert d.microsecond == 0 + assert d.tzinfo == zoneinfo.ZoneInfo('Europe/London') + + +def test_datetimetz_datetimetz_set_tzinfo_raises(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + with pytest.raises(AttributeError) as err: + d.tzinfo = None + assert err.value.args[0] == "attribute 'tzinfo' of 'datetime.datetime' objects is not writable" + + +@pytest.mark.skipif( + sys.version_info.minor == 9, + reason="Fails on Python 3.9 with \"Failed: DID NOT RAISE \"", +) +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + (2024, 7, 15, 10, 21, 14), + {}, + 'No time zone provided.', + ), + ( + (2024, 7, 15, 10, 21, 14), + {'tzinfo': None, }, + 'No time zone provided.', + ), + ) +) +def test_datetimetz_datetimetz_raises(args, kwargs, expected): + with pytest.raises(TypeError) as err: + d = datetimetz.datetimetz(*args, **kwargs) + print() + print(f'ERROR: {repr(d)}') + assert err.value.args[0] == expected + + +def test_datetimetz_datetimetz_equal(): + d_tz = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London')) + d = datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + assert d_tz == d + + +def test_datetime_datetime_equal_naive(): + d = datetime.datetime( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + d_no_tz = datetime.datetime(2024, 7, 15, 10, 21, 14) + assert d_no_tz != d + + +def test_datetimetz_datetimetz_equal_naive(): + d_tz = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + d = datetime.datetime(2024, 7, 15, 10, 21, 14) + assert d_tz != d + + +@pytest.mark.parametrize( + 'd_tz, d, expected', + ( + ( + datetimetz.datetimetz(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('Europe/London')), + datetime.datetime(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('Europe/London')), + datetime.timedelta(0), + ), + ( + datetimetz.datetimetz(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('Europe/London')), + datetime.datetime(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('America/New_York')), + datetime.timedelta(seconds=-5 * 60 * 60), + ), + ( + datetimetz.datetimetz(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('America/New_York')), + datetime.datetime(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('Europe/London')), + datetime.timedelta(seconds=5 * 60 * 60), + ), + ) +) +def test_datetimetz_datetimetz_subtract(d_tz, d, expected): + assert (d_tz - d) == expected + + +@pytest.mark.parametrize( + 'd_tz, d, expected', + ( + ( + datetimetz.datetimetz(2024, 7, 15, 10, 21, 14, tzinfo=zoneinfo.ZoneInfo('Europe/London')), + datetime.datetime(2024, 7, 15, 10, 21, 14), + '', + ), + ) +) +def test_datetimetz_datetimetz_subtract_raises(d_tz, d, expected): + with pytest.raises(TypeError) as err: + d_tz - d + assert err.value.args[0] == "can't subtract offset-naive and offset-aware datetimes" + + +def test_datetimetz_datetimetz_replace_year(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + d_replace = d.replace(year=2025) + print() + print(type(d_replace)) + assert type(d_replace) == datetimetz.datetimetz + assert d_replace.tzinfo is not None + assert d_replace.year == 2025 + assert d_replace == datetimetz.datetimetz( + 2025, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + + +def test_datetimetz_datetimetz_replace_raises_tzinfo(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + with pytest.raises(TypeError) as err: + d.replace(tzinfo=None) + print() + print(f'ERROR: {repr(d)}') + print(f'ERROR: {repr(d.tzinfo)}') + assert err.value.args[0] == 'No time zone provided.' + + +@pytest.mark.skipif(not (sys.version_info.minor <= 9), reason='Python 3.9') +def test_datetimetz_datetimetz_replace_raises_year_none_39_310(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + with pytest.raises(TypeError) as err: + d.replace(year=None) + assert err.value.args[0] == "an integer is required (got type NoneType)" + + +@pytest.mark.skipif(not (sys.version_info.minor >= 10), reason='Python 3.10+') +def test_datetimetz_datetimetz_replace_raises_year_none(): + d = datetimetz.datetimetz( + 2024, 7, 15, 10, 21, 14, + tzinfo=zoneinfo.ZoneInfo('Europe/London') + ) + with pytest.raises(TypeError) as err: + d.replace(year=None) + assert err.value.args[0] == "'NoneType' object cannot be interpreted as an integer" diff --git a/tests/unit/test_c_cpp.py b/tests/unit/test_c_cpp.py new file mode 100644 index 0000000..9d4a74c --- /dev/null +++ b/tests/unit/test_c_cpp.py @@ -0,0 +1,119 @@ +import datetime +import sys +import zoneinfo + +import psutil +import pytest + +from cPyExtPatt.cpp import placement_new +from cPyExtPatt.cpp import cUnicode + + +# (PythonExtPatt3.11_A) $ PythonExtensionPatterns git:(develop) ✗ python +# Python 3.11.6 (v3.11.6:8b6ee5ba3b, Oct 2 2023, 11:18:21) [Clang 13.0.0 (clang-1300.0.29.30)] on darwin +# Type "help", "copyright", "credits" or "license" for more information. +# >>> from cPyExtPatt.cpp import placement_new +# >>> dir(placement_new) +# ['CppCtorDtorInPyObject', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__'] +# >>> c = placement_new.CppCtorDtorInPyObject() +# -- CppCtorDtorInPyObject_new() +# Constructor at 0x10afea110 with argument "Default" +# Default constructor at 0x10afea110 with argument "Default" +# Initial self->Attr: Verbose object at 0x10afea110 m_str: "Default" +# Constructor at 0x60000043c060 with argument "pAttr" +# Initial self->pAttr: Verbose object at 0x60000043c060 m_str: "pAttr" +# >>> del c +# -- CppCtorDtorInPyObject_dealloc() +# self->Attr before delete: Verbose object at 0x10afea110 m_str: "Default" +# Destructor at 0x10afea110 m_str: "Default" +# self->pAttr before delete: Verbose object at 0x60000043c060 m_str: "pAttr" +# Destructor at 0x60000043c060 m_str: "pAttr" + +def test_placement_new(): + obj = placement_new.CppCtorDtorInPyObject() + del (obj) + + +@pytest.mark.parametrize( + 'count', + (1, 4, 8,) +) +def test_placement_new_memory(count): + """Tests repeated construction and destruction with a del call. + TODO: This is a flakey test measuring memory like this. + """ + proc = psutil.Process() + print() + rss_start = proc.memory_info().rss + print(f'RSS start: {rss_start:,d}') + # Python 3.10: 65_044_684 < 10_485_760 on occasion. + # Python 3.10: 280_023_244 < 10_485_760 on occasion. + rss_margin = 300 * 1024 * 1024 + for i in range(count): + obj = placement_new.CppCtorDtorInPyObject() + buffer_size = obj.buffer_size() + print(f'Buffer size: {buffer_size:,d}') + rss = proc.memory_info().rss + print(f' RSS new: {rss:,d} {rss - rss_start:+,d}') + assert abs(rss - rss_start - buffer_size) < rss_margin + del (obj) + rss = proc.memory_info().rss + print(f' RSS del: {rss:,d} {rss - rss_start:+,d}') + assert abs(rss - rss_start) < rss_margin + rss = proc.memory_info().rss + print(f' RSS end: {rss:,d} {rss - rss_start:+,d}') + assert abs(rss - rss_start) < rss_margin + + +@pytest.mark.parametrize( + 'count', + (1, 4, 8,) +) +def test_placement_new_memory_no_del(count): + """Tests repeated construction and destruction with no del call. + Within the loop the results are not really reproducible.""" + proc = psutil.Process() + print() + rss_start = proc.memory_info().rss + print(f'RSS start: {rss_start:,d}') + rss_margin = 3 * 60 * 1024 * 1024 + for i in range(count): + obj = placement_new.CppCtorDtorInPyObject() + buffer_size = obj.buffer_size() + print(f'Buffer size: {buffer_size:,d}') + rss = proc.memory_info().rss + print(f' RSS new: {rss:,d} {rss - rss_start:+,d}') + # assert abs(rss - rss_start - buffer_size) < rss_margin + rss = proc.memory_info().rss + print(f' RSS del: {rss:,d} {rss - rss_start:+,d}') + # assert abs(rss - rss_start) < (rss_margin + buffer_size) + rss = proc.memory_info().rss + print(f' RSS end: {rss:,d} {rss - rss_start:+,d}') + assert abs(rss - rss_start) < (rss_margin + buffer_size) + + +@pytest.mark.parametrize( + 'input, expected', + ( + ('String', 'String',), + ("a\xac\u1234\u20ac\U00008000", 'a¬ሴ€耀',), + ("a\xac\u1234\u20ac\U00018000", 'a¬ሴ€𘀀',), + ) +) +def test_unicode_to_string_and_back(input, expected): + result = cUnicode.unicode_to_string_and_back(input) + assert result == expected + + +@pytest.mark.parametrize( + 'input, expected', + ( + ('String', 'String',), + (b'String', b'String',), + (bytearray('String', 'ascii'), bytearray(b'String'),), + ) +) +def test_py_object_to_string_and_back(input, expected): + result = cUnicode.py_object_to_string_and_back(input) + assert result == expected + diff --git a/tests/unit/test_c_ctxmgr.py b/tests/unit/test_c_ctxmgr.py new file mode 100644 index 0000000..a71068d --- /dev/null +++ b/tests/unit/test_c_ctxmgr.py @@ -0,0 +1,56 @@ +import sys + +import psutil +import pytest + +from cPyExtPatt import cCtxMgr + + +def test_module_dir(): + assert dir(cCtxMgr) == ['BUFFER_LENGTH', 'ContextManager', '__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__'] + + +def test_module_BUFFER_LENGTH(): + assert cCtxMgr.BUFFER_LENGTH == 128 * 1024**2 + + +def test_very_simple(): + print() + with cCtxMgr.ContextManager(): + pass + + +def test_simple(): + print() + with cCtxMgr.ContextManager() as context: + assert sys.getrefcount(context) == 3 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == cCtxMgr.BUFFER_LENGTH + assert sys.getrefcount(context) == 2 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == 0 + del context + + +def test_memory(): + proc = psutil.Process() + print() + print(f'RSS START: {proc.memory_info().rss:12,d}') + for i in range(8): + print(f'RSS START {i:5d}: {proc.memory_info().rss:12,d}') + with cCtxMgr.ContextManager() as context: + print(f'RSS START CTX: {proc.memory_info().rss:12,d}') + # Does not work in the debugger due to introspection. + # assert sys.getrefcount(context) == 3 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == cCtxMgr.BUFFER_LENGTH + print(f'RSS END CTX: {proc.memory_info().rss:12,d}') + # Does not work in the debugger due to introspection. + # assert sys.getrefcount(context) == 2 + assert context.len_buffer_lifetime() == cCtxMgr.BUFFER_LENGTH + assert context.len_buffer_context() == 0 + del context + print(f'RSS END {i:5d}: {proc.memory_info().rss:12,d}') + print(f'RSS END: {proc.memory_info().rss:12,d}') + # assert 0 diff --git a/tests/unit/test_c_custom_pickle.py b/tests/unit/test_c_custom_pickle.py new file mode 100644 index 0000000..0bf5deb --- /dev/null +++ b/tests/unit/test_c_custom_pickle.py @@ -0,0 +1,84 @@ +import io +import pickle +import pickletools +import sys + +import pytest + +from cPyExtPatt import cPickle + + +def test_module_dir(): + assert dir(cPickle) == ['Custom', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__'] + + +ARGS_FOR_CUSTOM_CLASS = ('FIRST', 'LAST', 11) +PICKLE_BYTES_FOR_CUSTOM_CLASS = (b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x12cPyExtPatt.cPickle\x94' + b'\x8c\x06Custom\x94\x93\x94)\x81\x94}\x94(\x8c\x05first\x94\x8c\x05FIRST' + b'\x94\x8c\x04last\x94\x8c\x04LAST\x94\x8c\x06number\x94K\x0b\x8c\x0f_pickle_' + b'version\x94K\x01ub.') + + +def test_pickle_getstate(): + custom = cPickle.Custom(*ARGS_FOR_CUSTOM_CLASS) + pickled_value = pickle.dumps(custom) + print() + print(f'Pickled original is {pickled_value}') + assert pickled_value == PICKLE_BYTES_FOR_CUSTOM_CLASS + # result = pickle.loads(pickled_value) + + +def test_pickle_setstate(): + custom = pickle.loads(PICKLE_BYTES_FOR_CUSTOM_CLASS) + assert custom.first == 'FIRST' + assert custom.last == 'LAST' + assert custom.number == 11 + + +def test_pickle_round_trip(): + custom = cPickle.Custom(*ARGS_FOR_CUSTOM_CLASS) + pickled_value = pickle.dumps(custom) + result = pickle.loads(pickled_value) + assert id(result) != id(custom) + + +def test_pickletools(): + outfile = io.StringIO() + pickletools.dis(PICKLE_BYTES_FOR_CUSTOM_CLASS, out=outfile, annotate=1) + result = outfile.getvalue() + # print() + # print(result) + expected = """ 0: \\x80 PROTO 4 Protocol version indicator. + 2: \\x95 FRAME 102 Indicate the beginning of a new frame. + 11: \\x8c SHORT_BINUNICODE 'cPyExtPatt.cPickle' Push a Python Unicode string object. + 31: \\x94 MEMOIZE (as 0) Store the stack top into the memo. The stack is not popped. + 32: \\x8c SHORT_BINUNICODE 'Custom' Push a Python Unicode string object. + 40: \\x94 MEMOIZE (as 1) Store the stack top into the memo. The stack is not popped. + 41: \\x93 STACK_GLOBAL Push a global object (module.attr) on the stack. + 42: \\x94 MEMOIZE (as 2) Store the stack top into the memo. The stack is not popped. + 43: ) EMPTY_TUPLE Push an empty tuple. + 44: \\x81 NEWOBJ Build an object instance. + 45: \\x94 MEMOIZE (as 3) Store the stack top into the memo. The stack is not popped. + 46: } EMPTY_DICT Push an empty dict. + 47: \\x94 MEMOIZE (as 4) Store the stack top into the memo. The stack is not popped. + 48: ( MARK Push markobject onto the stack. + 49: \\x8c SHORT_BINUNICODE 'first' Push a Python Unicode string object. + 56: \\x94 MEMOIZE (as 5) Store the stack top into the memo. The stack is not popped. + 57: \\x8c SHORT_BINUNICODE 'FIRST' Push a Python Unicode string object. + 64: \\x94 MEMOIZE (as 6) Store the stack top into the memo. The stack is not popped. + 65: \\x8c SHORT_BINUNICODE 'last' Push a Python Unicode string object. + 71: \\x94 MEMOIZE (as 7) Store the stack top into the memo. The stack is not popped. + 72: \\x8c SHORT_BINUNICODE 'LAST' Push a Python Unicode string object. + 78: \\x94 MEMOIZE (as 8) Store the stack top into the memo. The stack is not popped. + 79: \\x8c SHORT_BINUNICODE 'number' Push a Python Unicode string object. + 87: \\x94 MEMOIZE (as 9) Store the stack top into the memo. The stack is not popped. + 88: K BININT1 11 Push a one-byte unsigned integer. + 90: \\x8c SHORT_BINUNICODE '_pickle_version' Push a Python Unicode string object. + 107: \\x94 MEMOIZE (as 10) Store the stack top into the memo. The stack is not popped. + 108: K BININT1 1 Push a one-byte unsigned integer. + 110: u SETITEMS (MARK at 48) Add an arbitrary number of key+value pairs to an existing dict. + 111: b BUILD Finish building an object, via __setstate__ or dict update. + 112: . STOP Stop the unpickling machine. +highest protocol among opcodes = 4 +""" + assert result == expected diff --git a/tests/unit/test_c_exceptions.py b/tests/unit/test_c_exceptions.py new file mode 100644 index 0000000..1476d34 --- /dev/null +++ b/tests/unit/test_c_exceptions.py @@ -0,0 +1,94 @@ +import sys + +import pytest + +from cPyExtPatt import cExceptions + + +def test_raise_error(): + with pytest.raises(ValueError) as err: + cExceptions.raise_error() + assert err.value.args[0] == 'Ooops.' + + +@pytest.mark.skipif(sys.version_info.minor > 9, reason='Python <= 3.9') +def test_raise_error_bad_old(): + with pytest.raises(SystemError) as err: + cExceptions.raise_error_bad() + assert err.value.args[0] == ' returned NULL without setting an error' + + +@pytest.mark.skipif(sys.version_info.minor <= 9, reason='Python > 3.9') +def test_raise_error_bad(): + with pytest.raises(SystemError) as err: + cExceptions.raise_error_bad() + assert err.value.args[0] == ' returned NULL without setting an exception' + + +def test_raise_error_fmt(): + with pytest.raises(ValueError) as err: + cExceptions.raise_error_fmt() + assert err.value.args[0] == 'Can not read 12 bytes when offset 25 in byte length 32.' + + +def test_raise_error_overwrite(): + with pytest.raises(ValueError) as err: + cExceptions.raise_error_overwrite() + assert err.value.args[0] == 'ERROR: raise_error_overwrite()' + + +@pytest.mark.skipif(sys.version_info.minor > 9, reason='Python <= 3.9') +def test_raise_error_silent_old(): + with pytest.raises(SystemError) as err: + cExceptions.raise_error_silent() + assert err.value.args[0] == ' returned a result with an error set' + + +@pytest.mark.skipif(sys.version_info.minor <= 9, reason='Python > 3.9') +def test_raise_error_silent(): + with pytest.raises(SystemError) as err: + cExceptions.raise_error_silent() + assert err.value.args[0] == ' returned a result with an exception set' + + +def test_raise_error_silent_test(): + cExceptions.raise_error_silent_test() + + +def test_ExceptionBase_exists(): + exception = cExceptions.ExceptionBase('FOO') + assert exception.args[0] == 'FOO' + assert str(exception.__class__.__mro__) == ( + "(" + "," + " ," + " ," + " " + ")" + ) + + +def test_SpecialsiedError_exists(): + exception = cExceptions.SpecialisedError('FOO') + assert exception.args[0] == 'FOO' + assert str(exception.__class__.__mro__) == ( + "(" + "," + " ," + " ," + " ," + " " + ")" + ) + + +def test_raise_exception_base(): + with pytest.raises(cExceptions.ExceptionBase) as err: + cExceptions.raise_exception_base() + assert err.value.args[0] == 'One 1 two 2 three 3.' + + +def test_raise_specialised_error(): + with pytest.raises(cExceptions.SpecialisedError) as err: + cExceptions.raise_specialised_error() + assert err.value.args[0] == 'One 1 two 2 three 3.' diff --git a/tests/unit/test_c_file.py b/tests/unit/test_c_file.py new file mode 100644 index 0000000..a33adfb --- /dev/null +++ b/tests/unit/test_c_file.py @@ -0,0 +1,141 @@ +import io +import sys +import pathlib +import typing + +import pytest + +from cPyExtPatt import cFile + + +@pytest.mark.parametrize( + 'arg, expected', + ( + ('~/foo/bar.txt', '~/foo/bar.txt',), + (pathlib.Path('~/foo/bar.txt'), '~/foo/bar.txt',), + ) +) +def test_parse_filesystem_argument(arg, expected): + assert cFile.parse_filesystem_argument(arg) == expected + + +@pytest.mark.parametrize( + 'arg, expected', + ( + ('~/foo/bar.txt', str,), + ) +) +def test_parse_filesystem_argument_return_type(arg, expected): + assert type(cFile.parse_filesystem_argument(arg)) == expected + + +@pytest.mark.skipif(not (sys.version_info.minor >= 7), reason='Python 3.7+') +@pytest.mark.parametrize( + 'arg, expected', + ( + # Number of arguments. + (None, "function missing required argument 'path' (pos 1)"), + ([1, 2.9], 'expected str, bytes or os.PathLike object, not list'), + ) +) +def test_parse_filesystem_argument_raises(arg, expected): + with pytest.raises(TypeError) as err: + if arg is None: + cFile.parse_filesystem_argument() + else: + cFile.parse_filesystem_argument(arg) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(not (sys.version_info.minor < 7), reason='Python < 3.7') +@pytest.mark.parametrize( + 'arg, expected', + ( + # Number of arguments. + (None, "Required argument 'path' (pos 1) not found"), + ([1, 2.9], 'expected str, bytes or os.PathLike object, not list'), + ) +) +def test_parse_filesystem_argument_raises_pre_37(arg, expected): + with pytest.raises(TypeError) as err: + if arg is None: + cFile.parse_filesystem_argument() + else: + cFile.parse_filesystem_argument(arg) + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'file_object, size, expected', + ( + (io.BytesIO(b'Some bytes.'), 4, b'Some'), + (io.BytesIO(b'Some bytes.'), None, b'Some bytes.'), + ) +) +def test_read_python_file_to_c(file_object, size, expected): + if size is None: + result = cFile.read_python_file_to_c(file_object) + else: + result = cFile.read_python_file_to_c(file_object, size) + assert result == expected + + +@pytest.mark.parametrize( + 'bytes_to_write, expected', + ( + (b'Some bytes.', len(b'Some bytes.')), + (b'Some\0bytes.', len(b'Some\0bytes.')), + ) +) +def test_write_bytes_to_python_string_file(bytes_to_write, expected): + file = io.StringIO() + result = cFile.write_bytes_to_python_file(bytes_to_write, file) + assert result == expected + + +@pytest.mark.parametrize( + 'bytes_to_write, expected', + ( + ('Some string.', "a bytes-like object is required, not 'str'"), + ) +) +def test_write_bytes_to_python_string_file_raises(bytes_to_write, expected): + file = io.StringIO() + with pytest.raises(TypeError) as err: + cFile.write_bytes_to_python_file(bytes_to_write, file) + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'bytes_to_write, expected', + ( + ('Some bytes.', "a bytes-like object is required, not 'str'"), + (b'Some bytes.', "a bytes-like object is required, not 'str'"), + ) +) +def test_write_bytes_to_python_bytes_file_raises(bytes_to_write, expected): + file = io.BytesIO() + with pytest.raises(TypeError) as err: + cFile.write_bytes_to_python_file(bytes_to_write, file) + assert err.value.args[0] == expected + +# TODO: Fix this. Why is position 420 when it is a 25 character write? String termination? +@pytest.mark.skip( + reason=( + "Fails on Python 3.9 and 3.11 with" + " \"UnicodeDecodeError: 'ascii' codec can't decode byte 0xe3 in position 420: ordinal not in range(128)\"" + ), +) +def test_wrap_python_file(): + file = io.BytesIO() + result = cFile.wrap_python_file(file) + print() + print(' Result '.center(75, '-')) + print(result.decode('ascii')) + print(' Result DONE '.center(75, '-')) + print(' file.getvalue() '.center(75, '-')) + get_value = file.getvalue() + print(get_value) + print(' file.getvalue() DONE '.center(75, '-')) + assert get_value == b'Test write to python file' + \ No newline at end of file diff --git a/tests/unit/test_c_iterators.py b/tests/unit/test_c_iterators.py new file mode 100644 index 0000000..9d569fb --- /dev/null +++ b/tests/unit/test_c_iterators.py @@ -0,0 +1,295 @@ +import sys + +import pytest + +from cPyExtPatt.Iterators import cIterator + + +def test_c_iterator_dir(): + result = dir(cIterator) + assert result == ['SequenceOfLong', + 'SequenceOfLongIterator', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'iterate_and_print'] + + +@pytest.mark.skipif(not (sys.version_info.minor < 11), reason='Python < 3.11') +def test_c_iterator_sequence_of_long_dir_pre_311(): + result = dir(cIterator.SequenceOfLong) + assert result == ['__class__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'size'] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python >= 3.11') +def test_c_iterator_sequence_of_long_dir_311_plus(): + result = dir(cIterator.SequenceOfLong) + assert result == ['__class__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'size'] + + +@pytest.mark.skipif(not (sys.version_info.minor < 11), reason='Python < 3.11') +def test_c_iterator_sequence_of_long_iterator_dir_pre_311(): + result = dir(cIterator.SequenceOfLongIterator) + assert result == ['__class__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__lt__', + '__ne__', + '__new__', + '__next__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__'] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python >= 3.11') +def test_c_iterator_sequence_of_long_iterator_dir_311_plus(): + result = dir(cIterator.SequenceOfLongIterator) + assert result == ['__class__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__lt__', + '__ne__', + '__new__', + '__next__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__'] + + +def test_c_iterator_ctor(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + assert sequence + assert type(sequence) is cIterator.SequenceOfLong + assert sequence.size() == 3 + + +def test_c_iterator_ctor_iter(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + iterator = iter(sequence) + assert iterator + assert type(iterator) is cIterator.SequenceOfLongIterator + + +def test_c_iterator_ctor_iter_for(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = [v for v in sequence] + assert result == [1, 7, 4] + + +def test_c_iterator_ctor_iter_del_original(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + iterator = iter(sequence) + del sequence + result = [v for v in iterator] + assert result == [1, 7, 4] + + +def test_c_iterator_ctor_iter_next(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + iterator = iter(sequence) + assert next(iterator) == 1 + assert next(iterator) == 7 + assert next(iterator) == 4 + + +def test_c_iterator_ctor_iter_next_raises(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + iterator = iter(sequence) + assert next(iterator) == 1 + assert next(iterator) == 7 + assert next(iterator) == 4 + with pytest.raises(StopIteration): + next(iterator) + + +def yield_from_an_iterator_times_two(iterator): + for value in iterator: + yield 2 * value + + +def test_c_iterator_yield_forward(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + iterator = iter(sequence) + result = [] + for v in yield_from_an_iterator_times_two(iterator): + result.append(v) + assert result == [2, 14, 8] + + +def test_c_iterator_sorted(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = sorted(sequence) + print() + print(result) + assert result == [1, 4, 7, ] + original = [v for v in sequence] + assert original == [1, 7, 4, ] + + +def test_c_iterator_reversed(): + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = reversed(sequence) + print() + print(result) + assert list(result) == [4, 7, 1,] + + +def test_c_iterator_generator_expression_sum(): + """https://docs.python.org/3/glossary.html#term-generator-expression""" + sequence = cIterator.SequenceOfLong([1, 7, 4]) + result = sum(v * 4 for v in sequence) + assert result == 4 * (1 + 7 + 4) + + +def test_modify_list_during_iteration_a(): + lst = list(range(8)) + print() + result = [] + for i, value in enumerate(lst): + result.append(value) + print(f'i={i} value={value}') + del lst[i] + assert result == [0, 2, 4, 6] + + +def test_modify_list_during_iteration_b(): + lst = list(range(8)) + print() + result = [] + for i, value in enumerate(lst): + result.append(value) + print(f'i={i} value={value}') + lst.pop() + assert result == [0, 1, 2, 3] + + +def test_modify_list_during_iteration_c(): + lst = list(range(8)) + print() + result = [] + for i, value in enumerate(lst): + result.append(value) + print(f'i={i} value={value}') + if i and i % 2 == 0: + lst.append(8 * i) + assert result == [0, 1, 2, 3, 4, 5, 6, 7, 16, 32, 48, 64, 80, 96] + + +@pytest.mark.parametrize( + 'arg, expected', + ( + ( + 'abc', + """iterate_and_print: +[0]: a +[1]: b +[2]: c +iterate_and_print: DONE +""" + ), + ) +) +def test_iterate_and_print(arg, expected, capfd): + cIterator.iterate_and_print(arg) + captured = capfd.readouterr() + assert captured.out == expected + + +@pytest.mark.parametrize( + 'arg, error', + ( + (1, "'int' object is not iterable"), + ) +) +def test_iterate_and_print_raises(arg, error): + with pytest.raises(TypeError) as err: + cIterator.iterate_and_print(arg) + assert err.value.args[0] == error diff --git a/tests/unit/test_c_logging.py b/tests/unit/test_c_logging.py new file mode 100644 index 0000000..0df6d53 --- /dev/null +++ b/tests/unit/test_c_logging.py @@ -0,0 +1,95 @@ +import logging +import sys + +import pytest + +from cPyExtPatt.Logging import cLogging + +#: Default log format (terse) +DEFAULT_OPT_LOG_FORMAT = '%(asctime)s %(process)d %(levelname)-8s %(message)s' +#: Default log format (verbose) +DEFAULT_OPT_LOG_FORMAT_VERBOSE = '%(asctime)s - %(filename)-16s#%(lineno)-4d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s' + +logging.basicConfig(level=logging.DEBUG, stream=sys.stderr, format=DEFAULT_OPT_LOG_FORMAT) + +logger = logging.getLogger(__file__) + + +def test_logging(): + logger.setLevel(logging.DEBUG) + logger.warning('Test warning message XXXX') + logger.debug('Test debug message XXXX') + # assert 0 + + +def test_c_logging_dir(): + result = dir(cLogging) + assert result == [ + 'CRITICAL', + 'DEBUG', + 'ERROR', + 'EXCEPTION', + 'INFO', + 'WARNING', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'c_file_line_function', + 'log', + 'py_file_line_function', + 'py_log_set_level', + ] + + +def test_c_logging_log(): + print() + cLogging.py_log_set_level(10) + result = cLogging.log(cLogging.ERROR, "Test log message") + assert result is None + + +def test_c_file_line_function_file(): + file, line, function = cLogging.c_file_line_function() + assert file == 'src/cpy/Logging/cLogging.c' + assert line == 143 + assert function == 'c_file_line_function' + + +def test_py_file_line_function_file(): + file, _line, _function = cLogging.py_file_line_function() + assert file == __file__ + + +def test_py_file_line_function_line(): + _file, line, _function = cLogging.py_file_line_function() + assert line == 67 + + +def test_py_file_line_function_function(): + _file, _line, function = cLogging.py_file_line_function() + assert function == 'test_py_file_line_function_function' + + +def main(): + logger.setLevel(logging.DEBUG) + logger.info('main') + logger.warning('Test warning message XXXX') + logger.debug('Test debug message XXXX') + logger.info('_test_logging') + test_logging() + print() + print(cLogging) + print(dir(cLogging)) + print() + logger.info('cLogging.log():') + cLogging.py_log_set_level(10) + cLogging.log(cLogging.ERROR, "cLogging.log(): Test log message") + + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/tests/unit/test_c_module_globals.py b/tests/unit/test_c_module_globals.py new file mode 100644 index 0000000..37451d1 --- /dev/null +++ b/tests/unit/test_c_module_globals.py @@ -0,0 +1,36 @@ +import pytest + +from cPyExtPatt import cModuleGlobals + + +def test_int(): + assert cModuleGlobals.INT == 42 + + +def test_str(): + assert cModuleGlobals.STR == 'String value' + + +def test_list(): + assert cModuleGlobals.LST == [66, 68, 73,] + + +def test_list_alter(): + assert cModuleGlobals.LST == [66, 68, 73,] + cModuleGlobals.LST.append(100) + assert cModuleGlobals.LST == [66, 68, 73, 100,] + assert cModuleGlobals.LST.pop(-1) == 100 + assert cModuleGlobals.LST == [66, 68, 73,] + + +def test_tuple(): + assert cModuleGlobals.TUP == (66, 68, 73,) + + +def test_map(): + assert cModuleGlobals.MAP == {b'123': 123, b'66': 66} + + +def test_print(): + cModuleGlobals.print() + # assert 0 diff --git a/tests/unit/test_c_object.py b/tests/unit/test_c_object.py new file mode 100644 index 0000000..171bb60 --- /dev/null +++ b/tests/unit/test_c_object.py @@ -0,0 +1,154 @@ +import sys + +import pytest + +from cPyExtPatt import cObject + + +def test_module_dir_not_3_10(): + assert dir(cObject) == ['Null', 'ObjectWithAttributes', 'Str', '__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__', 'error', ] + + +@pytest.mark.skipif(sys.version_info.minor > 10, reason='Python > 3.10') +def test_null_dir_3_10(): + null = cObject.Null() + assert dir(null) == ['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', + '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', + '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__'] + + +@pytest.mark.skipif(sys.version_info.minor <= 10, reason='Python <= 3.10') +def test_null_dir(): + null = cObject.Null() + assert dir(null) == ['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', + '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', + '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__'] + + +@pytest.mark.skipif(sys.version_info.minor <= 10, reason='Python <= 3.10') +def test_ObjectWithAttributes_dir_not_3_10(): + obj = cObject.ObjectWithAttributes() + # print() + # print(dir(obj)) + assert dir(obj) == ['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', + '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', + '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'demo', ] + + +@pytest.mark.skipif(sys.version_info.minor > 10, reason='Python > 3.10') +def test_ObjectWithAttributes_dir_3_10(): + obj = cObject.ObjectWithAttributes() + # print() + # print(dir(obj)) + assert dir(obj) == ['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', + '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', + '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'demo', ] + + +def test_ObjectWithAttributes_set_and_get(): + obj = cObject.ObjectWithAttributes() + obj.some_attr = 'Some attribute' + # print() + # print(obj) + # print(obj.some_attr) + assert hasattr(obj, 'some_attr') + assert obj.some_attr == 'Some attribute' + + +def test_ObjectWithAttributes_set_and_del(): + obj = cObject.ObjectWithAttributes() + obj.some_attr = 'Some attribute' + # print() + # print(obj) + # print(obj.some_attr) + assert hasattr(obj, 'some_attr') + delattr(obj, 'some_attr') + assert not hasattr(obj, 'some_attr') + with pytest.raises(AttributeError) as err: + obj.some_attr + assert err.value.args[0] == "'cObject.ObjectWithAttributes' object has no attribute 'some_attr'" + + +@pytest.mark.skipif(not (sys.version_info.minor < 7), reason='Python < 3.7') +def test_str_dir_pre_37(): + s = cObject.Str() + assert dir(s) == ['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', + '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', + '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', + '__mod__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmod__', + '__rmul__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'capitalize', 'casefold', + 'center', 'count', 'encode', 'endswith', 'expandtabs', 'find', 'format', 'format_map', 'index', + 'isalnum', 'isalpha', 'isdecimal', 'isdigit', 'isidentifier', 'islower', 'isnumeric', + 'isprintable', 'isspace', 'istitle', 'isupper', 'join', 'ljust', 'lower', 'lstrip', 'maketrans', + 'partition', 'replace', 'rfind', 'rindex', 'rjust', 'rpartition', + 'rsplit', 'rstrip', 'split', 'splitlines', 'startswith', 'strip', 'swapcase', 'title', + 'translate', 'upper', 'zfill'] + +@pytest.mark.skipif(not (7 <= sys.version_info.minor < 9), reason='Python 3.7, 3.8') +def test_str_dir_37_38(): + s = cObject.Str() + assert dir(s) == ['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', + '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', + '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', + '__mod__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmod__', + '__rmul__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'capitalize', 'casefold', + 'center', 'count', 'encode', 'endswith', 'expandtabs', 'find', 'format', 'format_map', 'index', + 'isalnum', 'isalpha', 'isascii', 'isdecimal', 'isdigit', 'isidentifier', 'islower', 'isnumeric', + 'isprintable', 'isspace', 'istitle', 'isupper', 'join', 'ljust', 'lower', 'lstrip', 'maketrans', + 'partition', 'replace', 'rfind', 'rindex', 'rjust', 'rpartition', + 'rsplit', 'rstrip', 'split', 'splitlines', 'startswith', 'strip', 'swapcase', 'title', + 'translate', 'upper', 'zfill'] + + +@pytest.mark.skipif(not (9 <= sys.version_info.minor < 11), reason='Python 3.9, 3.10') +def test_str_dir_39_310(): + s = cObject.Str() + assert dir(s) == ['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', + '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', + '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', + '__mod__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmod__', + '__rmul__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'capitalize', 'casefold', + 'center', 'count', 'encode', 'endswith', 'expandtabs', 'find', 'format', 'format_map', 'index', + 'isalnum', 'isalpha', 'isascii', 'isdecimal', 'isdigit', 'isidentifier', 'islower', 'isnumeric', + 'isprintable', 'isspace', 'istitle', 'isupper', 'join', 'ljust', 'lower', 'lstrip', 'maketrans', + 'partition', 'removeprefix', 'removesuffix', 'replace', 'rfind', 'rindex', 'rjust', 'rpartition', + 'rsplit', 'rstrip', 'split', 'splitlines', 'startswith', 'strip', 'swapcase', 'title', + 'translate', 'upper', 'zfill'] + +@pytest.mark.skipif(not (sys.version_info.minor > 10), reason='Python 3.10+') +def test_str_dir_3_10_plus(): + s = cObject.Str() + assert dir(s) == ['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', + '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', '__getstate__', + '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', + '__mod__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmod__', + '__rmul__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'capitalize', 'casefold', + 'center', 'count', 'encode', 'endswith', 'expandtabs', 'find', 'format', 'format_map', 'index', + 'isalnum', 'isalpha', 'isascii', 'isdecimal', 'isdigit', 'isidentifier', 'islower', 'isnumeric', + 'isprintable', 'isspace', 'istitle', 'isupper', 'join', 'ljust', 'lower', 'lstrip', 'maketrans', + 'partition', 'removeprefix', 'removesuffix', 'replace', 'rfind', 'rindex', 'rjust', 'rpartition', + 'rsplit', 'rstrip', 'split', 'splitlines', 'startswith', 'strip', 'swapcase', 'title', + 'translate', 'upper', 'zfill'] + + + +# Python +# @pytest.mark.skipif(not (sys.version_info.minor <= 8), reason='Python > 3.8') +# def test_str_dir_3_9_3_10(): +# s = cObject.Str() +# assert dir(s) == ['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', +# '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', +# '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', +# '__mod__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmod__', +# '__rmul__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'capitalize', 'casefold', +# 'center', 'count', 'encode', 'endswith', 'expandtabs', 'find', 'format', 'format_map', 'index', +# 'isalnum', 'isalpha', 'isascii', 'isdecimal', 'isdigit', 'isidentifier', 'islower', 'isnumeric', +# 'isprintable', 'isspace', 'istitle', 'isupper', 'join', 'ljust', 'lower', 'lstrip', 'maketrans', +# 'partition', 'replace', 'rfind', 'rindex', 'rjust', 'rpartition', +# 'rsplit', 'rstrip', 'split', 'splitlines', 'startswith', 'strip', 'swapcase', 'title', +# 'translate', 'upper', 'zfill'] diff --git a/tests/unit/test_c_parse_args.py b/tests/unit/test_c_parse_args.py new file mode 100644 index 0000000..55c3f27 --- /dev/null +++ b/tests/unit/test_c_parse_args.py @@ -0,0 +1,439 @@ +import sys + +import pytest + +from cPyExtPatt import cParseArgs + + +def test_module_dir(): + assert dir(cParseArgs) == ['__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__', 'parse_args', + 'parse_args_kwargs', 'parse_args_with_function_conversion_to_c', + 'parse_args_with_immutable_defaults', 'parse_args_with_mutable_defaults', + 'parse_default_bytes_object', + 'parse_no_args', 'parse_one_arg', 'parse_pos_only_kwd_only', ] + + +def test_parse_no_args(): + assert cParseArgs.parse_no_args() is None + + +@pytest.mark.skipif(not (sys.version_info.minor > 8), reason='Python > 3.8') +def test_parse_no_args_raises(): + with pytest.raises(TypeError) as err: + cParseArgs.parse_no_args(123) + assert err.value.args[0] == 'cPyExtPatt.cParseArgs.parse_no_args() takes no arguments (1 given)' + + +@pytest.mark.skipif(not (sys.version_info.minor <= 8), reason='Python <= 3.8') +def test_parse_no_args_raises_pre_39(): + with pytest.raises(TypeError) as err: + cParseArgs.parse_no_args(123) + assert err.value.args[0] == 'parse_no_args() takes no arguments (1 given)' + + +def test_parse_one_arg(): + assert cParseArgs.parse_one_arg(123) is None + + +@pytest.mark.skipif(not (sys.version_info.minor > 8), reason='Python > 3.8') +def test_parse_one_arg_raises(): + with pytest.raises(TypeError) as err: + cParseArgs.parse_one_arg(123, 456) + assert err.value.args[0] == 'cPyExtPatt.cParseArgs.parse_one_arg() takes exactly one argument (2 given)' + + +@pytest.mark.skipif(not (sys.version_info.minor <= 8), reason='Python <= 3.8') +def test_parse_one_arg_raises_pre_39(): + with pytest.raises(TypeError) as err: + cParseArgs.parse_one_arg(123, 456) + assert err.value.args[0] == 'parse_one_arg() takes exactly one argument (2 given)' + + +@pytest.mark.parametrize( + 'args, expected', + ( + ((b'bytes', 123), (b'bytes', 123, 'default_string')), + ((b'bytes', 123, 'local_string'), (b'bytes', 123, 'local_string')), + ) +) +def test_parse_args(args, expected): + assert cParseArgs.parse_args(*args) == expected + + +@pytest.mark.parametrize( + 'args, expected', + ( + # Number of arguments. + ((), 'function takes at least 2 arguments (0 given)'), + ((b'bytes',), 'function takes at least 2 arguments (1 given)'), + ((b'bytes', 123, 'str', 7), 'function takes at most 3 arguments (4 given)'), + # Type of arguments. + (('str', 456), 'argument 1 must be bytes, not str'), + ((b'bytes', 456, 456), 'argument 3 must be str, not int'), + ) +) +def test_parse_args_raises(args, expected): + """Signature is:: + + def parse_args(a: bytes, b: int, c: str = '') -> int: + """ + with pytest.raises(TypeError) as err: + cParseArgs.parse_args(*args) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(sys.version_info.minor > 9, reason='Python <= 3.9') +@pytest.mark.parametrize( + 'args, expected', + ( + ((b'bytes', 456.0), "integer argument expected, got float"), + ((b'bytes', '456'), "an integer is required (got type str)"), + ) +) +def test_parse_args_raises_conversion_old(args, expected): + """Signature is:: + + def parse_args(a: bytes, b: int, c: str = '') -> int: + """ + with pytest.raises(TypeError) as err: + cParseArgs.parse_args(*args) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(sys.version_info.minor <= 9, reason='Python > 3.9') +@pytest.mark.parametrize( + 'args, expected', + ( + ((b'bytes', 456.0), "'float' object cannot be interpreted as an integer"), + ((b'bytes', '456'), "'str' object cannot be interpreted as an integer"), + ) +) +def test_parse_args_raises_conversion(args, expected): + """Signature is:: + + def parse_args(a: bytes, b: int, c: str = '') -> int: + """ + with pytest.raises(TypeError) as err: + cParseArgs.parse_args(*args) + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ((b'b', 5), {}, b'bbbbb'), + (('b', 5), {}, 'bbbbb'), + ((b'b',), {'count': 5}, b'bbbbb'), + ((), {'sequence': b'b', 'count': 5}, b'bbbbb'), + (([1, 2, 3], 3), {}, [1, 2, 3, 1, 2, 3, 1, 2, 3]), + # NOTE: If count is zero then an empty sequence of given type is returned. + ((b'bytes', 0,), {}, b''), + ((b'b', 0,), {}, b''), + # NOTE: If count is absent then it defaults to 1. + ((b'bytes',), {}, b'bytes'), + ((b'b',), {}, b'b'), + # args/kwargs are None + (None, {'sequence': b'b', 'count': 5}, b'bbbbb'), + (('b', 5), None, 'bbbbb'), + ) +) +def test_parse_args_kwargs(args, kwargs, expected): + if args is None: + if kwargs is None: + assert cParseArgs.parse_args_kwargs() == expected + else: + assert cParseArgs.parse_args_kwargs(**kwargs) == expected + elif kwargs is None: + assert cParseArgs.parse_args_kwargs(*args) == expected + else: + assert cParseArgs.parse_args_kwargs(*args, **kwargs) == expected + # assert cParseArgs.parse_args_kwargs(*args, **kwargs) == expected + + +def test_parse_args_kwargs_examples(): + """Variations on the signature:: + + def parse_args_kwargs(sequence=typing.Sequence[typing.Any], count: int = 1) -> typing.Sequence[typing.Any]: + """ + assert cParseArgs.parse_args_kwargs([1, 2, 3], 2) == [1, 2, 3, 1, 2, 3] + assert cParseArgs.parse_args_kwargs([1, 2, 3], count=2) == [1, 2, 3, 1, 2, 3] + assert cParseArgs.parse_args_kwargs(sequence=[1, 2, 3], count=2) == [1, 2, 3, 1, 2, 3] + + +@pytest.mark.skipif(not (7 <= sys.version_info.minor), reason='Python 3.7+') +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ((), {}, "function missing required argument 'sequence' (pos 1)"), + ((5,), {'sequence': b'bytes', }, "argument for function given by name ('sequence') and position (1)"), + ((), {'count': 2}, "function missing required argument 'sequence' (pos 1)"), + ((), {'sequence': b'b', 'count': 5, 'foo': 27.2}, 'function takes at most 2 keyword arguments (3 given)'), + ((b'b',), {'count': 5, 'foo': 27.2}, 'function takes at most 2 arguments (3 given)'), + # args/kwargs are None + (None, {'count': 5, }, "function missing required argument 'sequence' (pos 1)"), + (None, None, "function missing required argument 'sequence' (pos 1)"), + ) +) +def test_parse_args_kwargs_raises(args, kwargs, expected): + with pytest.raises(TypeError) as err: + if args is None: + if kwargs is None: + cParseArgs.parse_args_kwargs() + else: + cParseArgs.parse_args_kwargs(**kwargs) + elif kwargs is None: + cParseArgs.parse_args_kwargs(*args) + else: + cParseArgs.parse_args_kwargs(*args, **kwargs) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(not (sys.version_info.minor < 7), reason='Python < 3.7') +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ((), {}, "Required argument 'sequence' (pos 1) not found"), + ((5,), {'sequence': b'bytes', }, "Argument given by name ('sequence') and position (1)"), + ((), {'count': 2}, "Required argument 'sequence' (pos 1) not found"), + ((), {'sequence': b'b', 'count': 5, 'foo': 27.2}, 'function takes at most 2 arguments (3 given)'), + ((b'b',), {'count': 5, 'foo': 27.2}, 'function takes at most 2 arguments (3 given)'), + # args/kwargs are None + (None, {'count': 5, }, "Required argument 'sequence' (pos 1) not found"), + (None, None, "Required argument 'sequence' (pos 1) not found"), + ) +) +def test_parse_args_kwargs_raises_pre_37(args, kwargs, expected): + with pytest.raises(TypeError) as err: + if args is None: + if kwargs is None: + cParseArgs.parse_args_kwargs() + else: + cParseArgs.parse_args_kwargs(**kwargs) + elif kwargs is None: + cParseArgs.parse_args_kwargs(*args) + else: + cParseArgs.parse_args_kwargs(*args, **kwargs) + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'args, expected', + ( + ( + (), + ('Hello world', ('Answer', 42)) + ), + ( + ('Some string',), + ('Some string', ('Answer', 42)) + ), + ( + ('Other string', ('Question', 456)), + ('Other string', ('Question', 456)), + ), + ), +) +def test_parse_args_with_immutable_defaults(args, expected): + assert cParseArgs.parse_args_with_immutable_defaults(*args) == expected + + +def py_parse_args_with_mutable_defaults(obj, obj_list=[]): + obj_list.append(obj) + return obj_list + + +def test_py_parse_args_with_mutable_defaults(): + """Tests the Python behaviour.""" + local_list = [] + # print() + # print(py_parse_args_with_mutable_defaults(1)) + # print(py_parse_args_with_mutable_defaults(2)) + # print(py_parse_args_with_mutable_defaults(3)) + # print() + # print(py_parse_args_with_mutable_defaults(-1, local_list)) + # print(py_parse_args_with_mutable_defaults(-2, local_list)) + # print() + # print(py_parse_args_with_mutable_defaults(4)) + + assert py_parse_args_with_mutable_defaults(1) == [1, ] + assert py_parse_args_with_mutable_defaults(2) == [1, 2, ] + assert py_parse_args_with_mutable_defaults(3) == [1, 2, 3, ] + assert py_parse_args_with_mutable_defaults(-1, local_list) == [-1, ] + assert py_parse_args_with_mutable_defaults(-2, local_list) == [-1, -2] + + assert py_parse_args_with_mutable_defaults(4) == [1, 2, 3, 4, ] + assert py_parse_args_with_mutable_defaults(5) == [1, 2, 3, 4, 5, ] + + assert py_parse_args_with_mutable_defaults(-3, local_list) == [-1, -2, -3, ] + + +def test_parse_args_with_mutable_defaults(): + """Tests the C extension behaviour.""" + local_list = [] + # print() + # print(cParseArgs.parse_args_with_mutable_defaults(1)) + # print(cParseArgs.parse_args_with_mutable_defaults(2)) + # print(cParseArgs.parse_args_with_mutable_defaults(3)) + # print() + # print(cParseArgs.parse_args_with_mutable_defaults(-1, local_list)) + # print(cParseArgs.parse_args_with_mutable_defaults(-2, local_list)) + # print() + # print(cParseArgs.parse_args_with_mutable_defaults(4)) + + assert cParseArgs.parse_args_with_mutable_defaults(1) == [1, ] + assert cParseArgs.parse_args_with_mutable_defaults(2) == [1, 2, ] + assert cParseArgs.parse_args_with_mutable_defaults(3) == [1, 2, 3, ] + assert cParseArgs.parse_args_with_mutable_defaults(-1, local_list) == [-1, ] + assert cParseArgs.parse_args_with_mutable_defaults(-2, local_list) == [-1, -2] + + assert cParseArgs.parse_args_with_mutable_defaults(4) == [1, 2, 3, 4, ] + assert cParseArgs.parse_args_with_mutable_defaults(5) == [1, 2, 3, 4, 5, ] + + assert cParseArgs.parse_args_with_mutable_defaults(-3, local_list) == [-1, -2, -3, ] + + +@pytest.mark.parametrize( + 'value, expected', + ( + (None, b"default"), + (b'local_value', b'local_value'), + ) +) +def test_parse_default_bytes_object(value, expected): + """Signature is:: + + def parse_default_bytes_object(b: bytes = b"default") -> bytes: + """ + if value is None: + result = cParseArgs.parse_default_bytes_object() + else: + result = cParseArgs.parse_default_bytes_object(value) + assert result == expected + + +# @pytest.mark.parametrize( +# 'args, expected', +# ( +# ((b'bytes', 123), (b'bytes', 123, 'default_string')), +# ((b'bytes', 123, 'local_string'), (b'bytes', 123, 'local_string')), +# ) +# ) +# def test_parse_pos_only_kwd_only(args, expected): +def test_parse_pos_only_kwd_only(): + """Signature is:: + + def parse_pos_only_kwd_only(pos1: str, pos2: int, /, pos_or_kwd: bytes, *, kwd1: float, kwd2: int) -> typing.Tuple[typing.Any, ...] + """ + result = cParseArgs.parse_pos_only_kwd_only('pos1', 12, b'pos_or_keyword') + print() + print(result) + assert result == ('pos1', 12, b'pos_or_keyword', 256.0, -421) + result = cParseArgs.parse_pos_only_kwd_only('pos1', 12, pos_or_kwd=b'pos_or_keyword') + print() + print(result) + assert result == ('pos1', 12, b'pos_or_keyword', 256.0, -421) + result = cParseArgs.parse_pos_only_kwd_only('pos1', 12, pos_or_kwd=b'pos_or_keyword', kwd1=8.0, kwd2=16) + print() + print(result) + assert result == ('pos1', 12, b'pos_or_keyword', 8.0, 16) + + +@pytest.mark.skipif(not (7 <= sys.version_info.minor), reason='Python 3.7+') +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + # Number of arguments. + ((), {}, 'function takes at least 2 positional arguments (0 given)'), + (('pos1', 12,), {}, "function missing required argument 'pos_or_kwd' (pos 3)"), + (('pos1', 12, b'pos_or_keyword', 'kwd1'), {}, 'function takes at most 3 positional arguments (4 given)'), + # See: test_parse_pos_only_kwd_only_raises_3_13 + # (('pos1', 12, b'pos_or_keyword'), {'pos1': 'pos1'}, + # "'pos1' is an invalid keyword argument for this function"), + ) +) +def test_parse_pos_only_kwd_only_raises(args, kwargs, expected): + """Signature is:: + + def parse_pos_only_kwd_only(pos1: str, pos2: int, /, pos_or_kwd: bytes, *, kwd1: float, kwd2: int): + """ + with pytest.raises(TypeError) as err: + cParseArgs.parse_pos_only_kwd_only(*args, **kwargs) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(not (sys.version_info.minor < 7), reason='Python < 3.7') +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + # Number of arguments. + ((), {}, 'Function takes at least 2 positional arguments (0 given)'), + (('pos1', 12,), {}, "Required argument 'pos_or_kwd' (pos 3) not found"), + (('pos1', 12, b'pos_or_keyword', 'kwd1'), {}, + 'Function takes at most 3 positional arguments (4 given)'), + ) +) +def test_parse_pos_only_kwd_only_raises_pre_37(args, kwargs, expected): + """Signature is:: + + def parse_pos_only_kwd_only(pos1: str, pos2: int, /, pos_or_kwd: bytes, *, kwd1: float, kwd2: int): + """ + with pytest.raises(TypeError) as err: + cParseArgs.parse_pos_only_kwd_only(*args, **kwargs) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(sys.version_info.minor >= 13, reason='Python 3.13 changed the error message.') +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + ('pos1', 12, b'pos_or_keyword'), {'pos1': 'pos1'}, + "'pos1' is an invalid keyword argument for this function", + ), + ) +) +def test_parse_pos_only_kwd_only_raises_before_3_13(args, kwargs, expected): + with pytest.raises(TypeError) as err: + cParseArgs.parse_pos_only_kwd_only(*args, **kwargs) + assert err.value.args[0] == expected + + +@pytest.mark.skipif(sys.version_info.minor < 13, reason='Python 3.13 changed the error message.') +@pytest.mark.parametrize( + 'args, kwargs, expected', + ( + ( + ('pos1', 12, b'pos_or_keyword'), {'pos1': 'pos1'}, + "this function got an unexpected keyword argument 'pos1'", + ), + ) +) +def test_parse_pos_only_kwd_only_raises_after_3_13(args, kwargs, expected): + with pytest.raises(TypeError) as err: + cParseArgs.parse_pos_only_kwd_only(*args, **kwargs) + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'arg, expected', + ( + ([], 0), + ([3, 7], 10), + ) +) +def test_parse_args_with_function_conversion_to_c(arg, expected): + assert cParseArgs.parse_args_with_function_conversion_to_c(arg) == expected + + +@pytest.mark.parametrize( + 'arg, expected', + ( + # Number of arguments. + ((), 'check_list_of_longs(): First argument is not a list'), + ([1, 2.9], 'check_list_of_longs(): Item 1 is not a Python integer.'), + ) +) +def test_parse_args_with_function_conversion_to_c_raises(arg, expected): + with pytest.raises(TypeError) as err: + cParseArgs.parse_args_with_function_conversion_to_c(arg) + assert err.value.args[0] == expected diff --git a/tests/unit/test_c_parse_args_helper.py b/tests/unit/test_c_parse_args_helper.py new file mode 100644 index 0000000..879e6f0 --- /dev/null +++ b/tests/unit/test_c_parse_args_helper.py @@ -0,0 +1,276 @@ +""" +Legacy code, see src/cpy/cParseArgsHelper.cpp for comments. +""" +import sys + +import pytest + +from cPyExtPatt import cParseArgsHelper +from cPyExtPatt import cPyRefs + + +def test_module_dir(): + assert dir(cParseArgsHelper) == [ + '__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__', + 'parse_defaults_with_helper_class', + 'parse_defaults_with_helper_macro', + 'parse_mutable_defaults_with_helper_class', + 'parse_mutable_defaults_with_helper_macro', + ] + + +@pytest.mark.parametrize( + 'args, expected', + ( + ( + (), + ('utf-8', 1024, 8.0), + ), + ( + ('Encoding', 4219, 16.0), + ('Encoding', 4219, 16.0), + ), + ), +) +def test_parse_defaults_with_helper_macro(args, expected): + print() + assert cParseArgsHelper.parse_defaults_with_helper_macro(*args) == expected + + +@pytest.mark.parametrize( + 'args, expected', + ( + ( + (b'utf-8', 1024, 8.0), + 'encoding_m must be "str", not "bytes"', + ), + ( + ('utf-8', 1024.0, 8.0), + 'the_id_m must be "int", not "float"', + ), + ( + ('utf-8', 1024, 8), + 'log_interval_m must be "float", not "int"', + ), + ), +) +def test_parse_defaults_with_helper_macro_raises_type_error(args, expected): + with pytest.raises(TypeError) as err: + cParseArgsHelper.parse_defaults_with_helper_macro(*args) + assert err.value.args[0] == expected + + +def test_parse_mutable_defaults_with_helper_macro_python(): + """A local Python equivalent of cParseArgsHelper.parse_mutable_defaults_with_helper_macro().""" + + def parse_mutable_defaults_with_helper_macro(obj, default_list=[]): + default_list.append(obj) + return default_list + + result = parse_mutable_defaults_with_helper_macro(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = parse_mutable_defaults_with_helper_macro(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = parse_mutable_defaults_with_helper_macro(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_macro(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_macro(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = parse_mutable_defaults_with_helper_macro(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 + + +def test_parse_mutable_defaults_with_helper_macro_c(): + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_macro(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_macro(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = cParseArgsHelper.parse_mutable_defaults_with_helper_macro(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 + + +# @pytest.mark.parametrize( +# 'args, expected', +# ( +# ( +# (), +# (4, 4, 4), +# ), +# ( +# (), +# (4, 4, 4), +# ), +# ( +# (), +# (4, 4, 4), +# ), +# ( +# ('Encoding',), +# (6, 4, 4), +# ), +# ( +# ('Encoding', 4219,), +# (6, 5, 4), +# ), +# ( +# ('Encoding', 4219, 16.0), +# (6, 5, 4), +# ), +# ), +# ) +# def test_parse_defaults_with_helper_macro_ref_counts(args, expected): +# result = cParseArgsHelper.parse_defaults_with_helper_macro(*args) +# ref_counts = tuple([cPyRefs.ref_count(v) for v in result]) +# assert ref_counts == expected +# del result + + +@pytest.mark.parametrize( + 'args, expected', + ( + ( + (), + ('utf-8', 1024, 8.0), + ), + ( + ('Encoding', 4219, 16.0), + ('Encoding', 4219, 16.0), + ), + ), +) +def test_parse_defaults_with_helper_class(args, expected): + print() + assert cParseArgsHelper.parse_defaults_with_helper_class(*args) == expected + + +@pytest.mark.parametrize( + 'args, expected', + ( + ( + (b'utf-8', 1024, 8.0), + 'encoding_c must be "str", not "bytes"', + ), + ( + ('utf-8', 1024.0, 8.0), + 'the_id_c must be "int", not "float"', + ), + ( + ('utf-8', 1024, 8), + 'log_interval_c must be "float", not "int"', + ), + ), +) +def test_parse_defaults_with_helper_class_raises_type_error(args, expected): + with pytest.raises(TypeError) as err: + cParseArgsHelper.parse_defaults_with_helper_class(*args) + assert err.value.args[0] == expected + + +# @pytest.mark.parametrize( +# 'args, expected', +# ( +# ( +# (), +# (4, 4, 4), +# ), +# ( +# (), +# (4, 4, 4), +# ), +# ( +# (), +# (4, 4, 4), +# ), +# ( +# ('Encoding',), +# (6, 4, 4), +# ), +# ( +# ('Encoding', 4219,), +# (6, 5, 4), +# ), +# ( +# ('Encoding', 4219, 16.0), +# (6, 5, 4), +# ), +# ), +# ) +# def test_parse_defaults_with_helper_class_ref_counts(args, expected): +# result = cParseArgsHelper.parse_defaults_with_helper_class(*args) +# ref_counts = tuple([cPyRefs.ref_count(v) for v in result]) +# assert ref_counts == expected +# del result + +def test_parse_mutable_defaults_with_helper_class_python(): + """A local Python equivalent of cParseArgsHelper.parse_mutable_defaults_with_helper_class().""" + + def parse_mutable_defaults_with_helper_class(obj, default_list=[]): + default_list.append(obj) + return default_list + + result = parse_mutable_defaults_with_helper_class(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = parse_mutable_defaults_with_helper_class(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = parse_mutable_defaults_with_helper_class(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_class(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert parse_mutable_defaults_with_helper_class(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = parse_mutable_defaults_with_helper_class(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 + + +def test_parse_mutable_defaults_with_helper_class_c(): + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(1) + assert sys.getrefcount(result) == 3 + assert result == [1, ] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(2) + assert sys.getrefcount(result) == 3 + assert result == [1, 2] + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(3) + assert sys.getrefcount(result) == 3 + assert result == [1, 2, 3] + + local_list = [] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_class(10, local_list) == [10] + assert sys.getrefcount(local_list) == 2 + assert cParseArgsHelper.parse_mutable_defaults_with_helper_class(11, local_list) == [10, 11] + assert sys.getrefcount(local_list) == 2 + + result = cParseArgsHelper.parse_mutable_defaults_with_helper_class(4) + assert result == [1, 2, 3, 4] + assert sys.getrefcount(result) == 3 diff --git a/tests/unit/test_c_py_refs.py b/tests/unit/test_c_py_refs.py new file mode 100644 index 0000000..835f756 --- /dev/null +++ b/tests/unit/test_c_py_refs.py @@ -0,0 +1,45 @@ +import pytest + +from cPyExtPatt import cPyRefs + + +def test_module_dir(): + assert dir(cPyRefs) == ['__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'access_after_free', + 'dec_ref', + 'inc_ref', + 'leak_new_reference', + 'make_tuple', + 'pop_and_print_BAD', + 'pop_and_print_OK', + 'ref_count', + 'subtract_two_longs', + ] + + +def test_ref_count(): + s = ''.join(dir(cPyRefs)) + assert cPyRefs.ref_count(s) == 2 + + +def test_ref_count_inc(): + s = ''.join(dir(cPyRefs)) + original_refcount = cPyRefs.ref_count(s) + assert original_refcount == 2 + assert cPyRefs.inc_ref(s) == original_refcount + assert cPyRefs.ref_count(s) == original_refcount + 1 + assert cPyRefs.dec_ref(s) == original_refcount + 1 + assert cPyRefs.ref_count(s) == original_refcount + + +def test_subtract_two_longs(): + assert cPyRefs.subtract_two_longs() == (421 - 17) + + +def test_make_tuple(): + assert cPyRefs.make_tuple() == (1, 2, 'three') diff --git a/tests/unit/test_c_ref_count.py b/tests/unit/test_c_ref_count.py new file mode 100644 index 0000000..98f2119 --- /dev/null +++ b/tests/unit/test_c_ref_count.py @@ -0,0 +1,391 @@ +import sys + +import pytest + +from cPyExtPatt import cRefCount + + +@pytest.mark.skipif(not (sys.version_info.minor < 13), reason='Python pre 3.13') +def test_module_dir_pre_3_13(): + assert dir(cRefCount) == [ + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'dict_buildvalue_no_steals', + 'dict_no_steals', + 'dict_no_steals_decref_after_set', + 'list_buildvalue_steals', + 'list_steals', + 'set_no_steals', + 'set_no_steals_decref', + 'test_PyDict_GetItem', + 'test_PyDict_SetDefault_default_unused', + 'test_PyDict_SetDefault_default_used', + 'test_PyDict_SetItem_fails_not_a_dict', + 'test_PyDict_SetItem_fails_not_hashable', + 'test_PyDict_SetItem_increments', + 'test_PyList_Append', + 'test_PyList_Append_fails_NULL', + 'test_PyList_Append_fails_not_a_list', + 'test_PyList_Insert', + 'test_PyList_Insert_Is_Truncated', + 'test_PyList_Insert_Negative_Index', + 'test_PyList_Insert_fails_NULL', + 'test_PyList_Insert_fails_not_a_list', + 'test_PyList_Py_BuildValue', + 'test_PyList_SET_ITEM_NULL', + 'test_PyList_SET_ITEM_NULL_SET_ITEM', + 'test_PyList_SET_ITEM_replace_same', + 'test_PyList_SET_ITEM_steals', + 'test_PyList_SET_ITEM_steals_replace', + 'test_PyList_SetIem_NULL_SetItem', + 'test_PyList_SetItem_NULL', + 'test_PyList_SetItem_fails_not_a_list', + 'test_PyList_SetItem_fails_out_of_range', + 'test_PyList_SetItem_replace_same', + 'test_PyList_SetItem_steals', + 'test_PyList_SetItem_steals_replace', + 'test_PySet_Add', + 'test_PySet_Discard', + 'test_PySet_Pop', + 'test_PyTuple_Py_BuildValue', + 'test_PyTuple_Py_PyTuple_Pack', + 'test_PyTuple_SET_ITEM_NULL', + 'test_PyTuple_SET_ITEM_NULL_SET_ITEM', + 'test_PyTuple_SET_ITEM_replace_same', + 'test_PyTuple_SET_ITEM_steals', + 'test_PyTuple_SET_ITEM_steals_replace', + 'test_PyTuple_SetIem_NULL_SetItem', + 'test_PyTuple_SetItem_NULL', + 'test_PyTuple_SetItem_fails_not_a_tuple', + 'test_PyTuple_SetItem_fails_out_of_range', + 'test_PyTuple_SetItem_replace_same', + 'test_PyTuple_SetItem_steals', + 'test_PyTuple_SetItem_steals_replace', + 'tuple_buildvalue_steals', + 'tuple_steals' + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python 3.13+') +def test_module_dir_3_13(): + assert dir(cRefCount) == [ + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'dict_buildvalue_no_steals', + 'dict_no_steals', + 'dict_no_steals_decref_after_set', + 'list_buildvalue_steals', + 'list_steals', + 'set_no_steals', + 'set_no_steals_decref', + 'test_PyDict_GetItem', + 'test_PyDict_Pop_key_absent', + 'test_PyDict_Pop_key_present', + 'test_PyDict_SetDefaultRef_default_unused', + 'test_PyDict_SetDefaultRef_default_used', + 'test_PyDict_SetDefault_default_unused', + 'test_PyDict_SetDefault_default_used', + 'test_PyDict_SetItem_fails_not_a_dict', + 'test_PyDict_SetItem_fails_not_hashable', + 'test_PyDict_SetItem_increments', + 'test_PyList_Append', + 'test_PyList_Append_fails_NULL', + 'test_PyList_Append_fails_not_a_list', + 'test_PyList_Insert', + 'test_PyList_Insert_Is_Truncated', + 'test_PyList_Insert_Negative_Index', + 'test_PyList_Insert_fails_NULL', + 'test_PyList_Insert_fails_not_a_list', + 'test_PyList_Py_BuildValue', + 'test_PyList_SET_ITEM_NULL', + 'test_PyList_SET_ITEM_NULL_SET_ITEM', + 'test_PyList_SET_ITEM_replace_same', + 'test_PyList_SET_ITEM_steals', + 'test_PyList_SET_ITEM_steals_replace', + 'test_PyList_SetIem_NULL_SetItem', + 'test_PyList_SetItem_NULL', + 'test_PyList_SetItem_fails_not_a_list', + 'test_PyList_SetItem_fails_out_of_range', + 'test_PyList_SetItem_replace_same', + 'test_PyList_SetItem_steals', + 'test_PyList_SetItem_steals_replace', + 'test_PySet_Add', + 'test_PySet_Discard', + 'test_PySet_Pop', + 'test_PyTuple_Py_BuildValue', + 'test_PyTuple_Py_PyTuple_Pack', + 'test_PyTuple_SET_ITEM_NULL', + 'test_PyTuple_SET_ITEM_NULL_SET_ITEM', + 'test_PyTuple_SET_ITEM_replace_same', + 'test_PyTuple_SET_ITEM_steals', + 'test_PyTuple_SET_ITEM_steals_replace', + 'test_PyTuple_SetIem_NULL_SetItem', + 'test_PyTuple_SetItem_NULL', + 'test_PyTuple_SetItem_fails_not_a_tuple', + 'test_PyTuple_SetItem_fails_out_of_range', + 'test_PyTuple_SetItem_replace_same', + 'test_PyTuple_SetItem_steals', + 'test_PyTuple_SetItem_steals_replace', + 'tuple_buildvalue_steals', + 'tuple_steals' + ] + + +def test_c_ref_count_tuple_steals(): + assert cRefCount.tuple_steals() == 0 + + +def test_c_ref_count_tuple_buildvalue_steals(): + assert cRefCount.tuple_buildvalue_steals() == 0 + + +def test_c_ref_count_list_steals(): + assert cRefCount.list_steals() == 0 + + +def test_c_ref_count_list_buildvalue_steals(): + assert cRefCount.list_buildvalue_steals() == 0 + + +def test_c_ref_count_set_no_steals(): + assert cRefCount.set_no_steals() == 0 + + +def test_c_ref_count_set_no_steals_decref(): + assert cRefCount.set_no_steals_decref() == 0 + + +def test_c_ref_count_dict_no_steals(): + assert cRefCount.dict_no_steals() == 0 + + +def test_c_ref_count_dict_no_steals_decref_after_set(): + assert cRefCount.dict_no_steals_decref_after_set() == 0 + + +def test_c_ref_count_dict_buildvalue_no_steals(): + assert cRefCount.dict_buildvalue_no_steals() == 0 + + +def test_PyTuple_SetItem_steals(): + assert cRefCount.test_PyTuple_SetItem_steals() == 0 + + +def test_PyTuple_SET_ITEM_steals(): + assert cRefCount.test_PyTuple_SET_ITEM_steals() == 0 + + +def test_PyTuple_SetItem_steals_replace(): + assert cRefCount.test_PyTuple_SetItem_steals_replace() == 0 + + +def test_PyTuple_SET_ITEM_steals_replace(): + assert cRefCount.test_PyTuple_SET_ITEM_steals_replace() == 0 + + +def test_PyTuple_SetItem_NULL(): + assert cRefCount.test_PyTuple_SetItem_NULL() == 0 + + +def test_PyTuple_SET_ITEM_NULL(): + assert cRefCount.test_PyTuple_SET_ITEM_NULL() == 0 + + +def test_PyTuple_SetIem_NULL_SetItem(): + assert cRefCount.test_PyTuple_SetIem_NULL_SetItem() == 0 + + +def test_PyTuple_SET_ITEM_NULL_SET_ITEM(): + assert cRefCount.test_PyTuple_SET_ITEM_NULL_SET_ITEM() == 0 + + +def test_PyTuple_SetItem_fails_not_a_tuple(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyTuple_SetItem_fails_not_a_tuple() + assert err.value.args[0].endswith('bad argument to internal function') + + +def test_PyTuple_SetItem_fails_out_of_range(): + with pytest.raises(IndexError) as err: + cRefCount.test_PyTuple_SetItem_fails_out_of_range() + assert err.value.args[0] == 'tuple assignment index out of range' + + +def test_PyTuple_SetItem_replace_same(): + assert cRefCount.test_PyTuple_SetItem_replace_same() == 0 + + +def test_PyTuple_SET_ITEM_replace_same(): + assert cRefCount.test_PyTuple_SET_ITEM_replace_same() == 0 + + +def test_PyTuple_Py_PyTuple_Pack(): + assert cRefCount.test_PyTuple_Py_PyTuple_Pack() == 0 + + +def test_PyTuple_Py_BuildValue(): + assert cRefCount.test_PyTuple_Py_BuildValue() == 0 + + +def test_PyList_SetItem_steals(): + assert cRefCount.test_PyList_SetItem_steals() == 0 + + +def test_PyList_SET_ITEM_steals(): + assert cRefCount.test_PyList_SET_ITEM_steals() == 0 + + +def test_PyList_SetItem_steals_replace(): + assert cRefCount.test_PyList_SetItem_steals_replace() == 0 + + +def test_PyList_SET_ITEM_steals_replace(): + assert cRefCount.test_PyList_SET_ITEM_steals_replace() == 0 + + +def test_PyList_SetItem_NULL(): + assert cRefCount.test_PyList_SetItem_NULL() == 0 + + +def test_PyList_SET_ITEM_NULL(): + assert cRefCount.test_PyList_SET_ITEM_NULL() == 0 + + +def test_PyList_SetIem_NULL_SetItem(): + assert cRefCount.test_PyList_SetIem_NULL_SetItem() == 0 + + +def test_PyList_SET_ITEM_NULL_SET_ITEM(): + assert cRefCount.test_PyList_SET_ITEM_NULL_SET_ITEM() == 0 + + +def test_PyList_SetItem_fails_not_a_list(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyList_SetItem_fails_not_a_list() + assert err.value.args[0].endswith('bad argument to internal function') + + +def test_PyList_SetItem_fails_out_of_range(): + with pytest.raises(IndexError) as err: + cRefCount.test_PyList_SetItem_fails_out_of_range() + assert err.value.args[0] == 'list assignment index out of range' + + +def test_PyList_Append(): + assert cRefCount.test_PyList_Append() == 0 + + +def test_PyList_Append_fails_not_a_list(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyList_Append_fails_not_a_list() + assert err.value.args[0].endswith(' bad argument to internal function') + + +def test_PyList_Append_fails_NULL(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyList_Append_fails_NULL() + assert err.value.args[0].endswith(' bad argument to internal function') + + +def test_PyList_Insert(): + assert cRefCount.test_PyList_Insert() == 0 + + +def test_PyList_Insert_Is_Truncated(): + assert cRefCount.test_PyList_Insert_Is_Truncated() == 0 + + +def test_PyList_Insert_Negative_Index(): + assert cRefCount.test_PyList_Insert_Negative_Index() == 0 + + +def test_PyList_Insert_fails_not_a_list(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyList_Insert_fails_not_a_list() + assert err.value.args[0].endswith(' bad argument to internal function') + + +def test_PyList_Insert_fails_NULL(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyList_Insert_fails_NULL() + assert err.value.args[0].endswith(' bad argument to internal function') + + +def test_PyList_SetItem_replace_same(): + assert cRefCount.test_PyList_SetItem_replace_same() == 0 + + +def test_PyList_SET_ITEM_replace_same(): + assert cRefCount.test_PyList_SET_ITEM_replace_same() == 0 + + +def test_PyList_Py_BuildValue(): + assert cRefCount.test_PyList_Py_BuildValue() == 0 + + +def test_PyDict_SetItem_increments(): + assert cRefCount.test_PyDict_SetItem_increments() == 0 + + +def test_PyDict_SetItem_fails_not_a_dict(): + with pytest.raises(SystemError) as err: + cRefCount.test_PyDict_SetItem_fails_not_a_dict() + assert err.value.args[0].endswith('bad argument to internal function') + + +def test_PyDict_SetItem_fails_not_hashable(): + with pytest.raises(TypeError) as err: + cRefCount.test_PyDict_SetItem_fails_not_hashable() + assert err.value.args[0] == "unhashable type: 'list'" + + +def test_PyDict_SetDefault_default_unused(): + assert cRefCount.test_PyDict_SetDefault_default_unused() == 0 + + +def test_PyDict_SetDefault_default_used(): + assert cRefCount.test_PyDict_SetDefault_default_used() == 0 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python 3.13+') +def test_PyDict_SetDefaultRef_default_unused(): + assert cRefCount.test_PyDict_SetDefaultRef_default_unused() == 0 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python 3.13+') +def test_PyDict_SetDefaultRef_default_used(): + assert cRefCount.test_PyDict_SetDefaultRef_default_used() == 0 + + +def test_test_PyDict_GetItem(): + assert cRefCount.test_PyDict_GetItem() == 0 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python 3.13+') +def test_PyDict_Pop_key_present(): + assert cRefCount.test_PyDict_Pop_key_present() == 0 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python 3.13+') +def test_PyDict_Pop_key_absent(): + assert cRefCount.test_PyDict_Pop_key_absent() == 0 + + +def test_PySet_Add(): + assert cRefCount.test_PySet_Add() == 0 + + +def test_PySet_Discard(): + assert cRefCount.test_PySet_Discard() == 0 + + +def test_PySet_Pop(): + assert cRefCount.test_PySet_Pop() == 0 diff --git a/tests/unit/test_c_seqobject.py b/tests/unit/test_c_seqobject.py new file mode 100644 index 0000000..f3cfe26 --- /dev/null +++ b/tests/unit/test_c_seqobject.py @@ -0,0 +1,349 @@ +import sys + +import pytest + +from cPyExtPatt import cSeqObject + + +def test_module_dir(): + assert dir(cSeqObject) == ['SequenceLongObject', '__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__', ] + + +@pytest.mark.skipif(not (sys.version_info.minor < 11), reason='Python < 3.11') +def test_SequenceLongObject_dir_pre_311(): + result = dir(cSeqObject.SequenceLongObject) + assert result == [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python >= 3.11') +def test_SequenceLongObject_dir_311_plus(): + result = dir(cSeqObject.SequenceLongObject) + assert result == [ + '__add__', + '__class__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getstate__', # New + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + ] + + +def test_SequenceLongObject_len(): + obj = cSeqObject.SequenceLongObject([7, 4, 1, ]) + assert len(obj) == 3 + + +def test_SequenceLongObject_concat(): + obj_a = cSeqObject.SequenceLongObject([7, 4, 1, ]) + # print(obj_a[4]) + # assert list(obj_a) == [7, 4, 1, ] + # assert [v for v in obj_a] == [7, 4, 1, ] + obj_b = cSeqObject.SequenceLongObject([70, 40, 100, ]) + assert id(obj_a) != id(obj_b) + obj = obj_a + obj_b + assert id(obj) != id(obj_a) + assert id(obj) != id(obj_b) + assert len(obj) == 6 + assert list(obj) == [7, 4, 1, ] + [70, 40, 100, ] + + +@pytest.mark.parametrize( + 'initial_sequence, count, expected', + ( + ( + [], 1, [], + ), + ( + [7, 4, 1, ], 0, [], + ), + ( + [7, 4, 1, ], -1, [], + ), + ( + [7, 4, 1, ], 1, [7, 4, 1, ], + ), + ( + [7, 4, 1, ], 2, [7, 4, 1, 7, 4, 1, ], + ), + ( + [7, 4, 1, ], 3, [7, 4, 1, 7, 4, 1, 7, 4, 1, ], + ), + ) +) +def test_SequenceLongObject_repeat(initial_sequence, count, expected): + obj_a = cSeqObject.SequenceLongObject(initial_sequence) + obj = obj_a * count + print() + assert id(obj_a) != id(obj) + assert list(obj) == expected + assert list(obj) == (list(obj_a) * count) + + +@pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ( + [7, 4, 1, ], 0, 7, + ), + ( + [7, 4, 1, ], 1, 4, + ), + ( + [7, 4, 1, ], 2, 1, + ), + ( + [7, 4, 1, ], -1, 1, + ), + ( + [7, 4, 1, ], -2, 4, + ), + ( + [7, 4, 1, ], -3, 7, + ), + ) +) +def test_SequenceLongObject_item(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + assert obj[index] == expected + + +@pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ( + [], 0, 'Index 0 is out of range for length 0', + ), + ( + [], -1, 'Index -1 is out of range for length 0', + ), + ( + [1, ], 2, 'Index 2 is out of range for length 1', + ), + ) +) +def test_SequenceLongObject_item_raises(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + with pytest.raises(IndexError) as err: + obj[index] + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'initial_sequence, index, value, expected', + ( + ( + [7, 4, 1, ], 0, 14, [14, 4, 1, ], + ), + ( + [7, 4, 1, ], -1, 14, [7, 4, 14, ], + ), + ( + [7, 4, 1, ], -2, 14, [7, 14, 1, ], + ), + ( + [7, 4, 1, ], -3, 14, [14, 4, 1, ], + ), + ) +) +def test_SequenceLongObject_setitem(initial_sequence, index, value, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + obj[index] = value + assert list(obj) == expected + + +@pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ( + [7, 4, 1, ], 3, 'Index 3 is out of range for length 3', + ), + ( + [7, 4, 1, ], -4, 'Index -4 is out of range for length 3', + ), + ) +) +def test_SequenceLongObject_setitem_raises(initial_sequence, index, expected): + print() + print(initial_sequence, index, expected) + obj = cSeqObject.SequenceLongObject(initial_sequence) + with pytest.raises(IndexError) as err: + obj[index] = 100 + print(list(obj)) + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ( + [7, ], 0, [], + ), + ( + [7, ], -1, [], + ), + ( + [7, 4, 1, ], 1, [7, 1, ], + ), + ( + [7, 4, ], 0, [4, ], + ), + ( + [7, 4, 1, ], -1, [7, 4, ], + ), + ( + [7, 4, 1, ], -2, [7, 1, ], + ), + ( + [7, 4, 1, ], -3, [4, 1, ], + ), + ) +) +def test_SequenceLongObject_delitem(initial_sequence, index, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + del obj[index] + assert list(obj) == expected + + +@pytest.mark.parametrize( + 'initial_sequence, index, expected', + ( + ( + [], 0, 'Index 0 is out of range for length 0', + ), + ( + [], -1, 'Index -1 is out of range for length 0', + ), + ( + [7, ], 1, 'Index 1 is out of range for length 1', + ), + ( + [7, ], -3, 'Index -3 is out of range for length 1', + ), + ) +) +def test_SequenceLongObject_delitem_raises(initial_sequence, index, expected): + print() + print(initial_sequence, index, expected) + obj = cSeqObject.SequenceLongObject(initial_sequence) + print(list(obj)) + with pytest.raises(IndexError) as err: + del obj[index] + assert err.value.args[0] == expected + + +@pytest.mark.parametrize( + 'initial_sequence, value, expected', + ( + ( + [7, ], 0, False, + ), + ( + [7, ], 7, True, + ), + ( + [1, 4, 7, ], 7, True, + ), + ) +) +def test_SequenceLongObject_contains(initial_sequence, value, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + result = value in obj + assert result == expected + + +def test_SequenceLongObject_concat_inplace(): + obj_a = cSeqObject.SequenceLongObject([7, 4, 1, ]) + obj_b = cSeqObject.SequenceLongObject([70, 40, 100, ]) + assert id(obj_a) != id(obj_b) + obj_a += obj_b + assert len(obj_a) == 6 + assert list(obj_a) == [7, 4, 1, ] + [70, 40, 100, ] + + +@pytest.mark.parametrize( + 'initial_sequence, count, expected', + ( + ( + [], 1, [], + ), + ( + [7, 4, 1, ], 0, [], + ), + ( + [7, 4, 1, ], -1, [], + ), + ( + [7, 4, 1, ], 1, [7, 4, 1, ], + ), + ( + [7, 4, 1, ], 2, [7, 4, 1, 7, 4, 1, ], + ), + ( + [7, 4, 1, ], 3, [7, 4, 1, 7, 4, 1, 7, 4, 1, ], + ), + ) +) +def test_SequenceLongObject_repeat_inplace(initial_sequence, count, expected): + obj = cSeqObject.SequenceLongObject(initial_sequence) + obj *= count + assert list(obj) == expected + assert list(obj) == (initial_sequence * count) diff --git a/tests/unit/test_c_simple_example.py b/tests/unit/test_c_simple_example.py new file mode 100644 index 0000000..8fd7f63 --- /dev/null +++ b/tests/unit/test_c_simple_example.py @@ -0,0 +1,34 @@ +import pytest + +from cPyExtPatt.SimpleExample import cFibA +from cPyExtPatt.SimpleExample import cFibB + + +@pytest.mark.parametrize( + 'index, expected', + ( + (1, 1,), + (2, 1,), + (3, 2,), + (8, 21,), + (30, 832040,), + ) +) +def test_cFibA_fibonacci(index, expected): + result = cFibA.fibonacci(index) + assert result == expected + + +@pytest.mark.parametrize( + 'index, expected', + ( + (1, 1,), + (2, 1,), + (3, 2,), + (8, 21,), + (30, 832040,), + ) +) +def test_cFibB_fibonacci(index, expected): + result = cFibB.fibonacci(index) + assert result == expected diff --git a/tests/unit/test_c_struct_sequence.py b/tests/unit/test_c_struct_sequence.py new file mode 100644 index 0000000..e00497b --- /dev/null +++ b/tests/unit/test_c_struct_sequence.py @@ -0,0 +1,770 @@ +import pprint +import sys + +import pytest + +from cPyExtPatt import cStructSequence + + +@pytest.mark.skipif(not (sys.version_info.minor < 11), reason='Python < 3.11') +def test_c_struct_sequence_dir_pre_3_11(): + result = dir(cStructSequence) + print() + print(result) + assert result == [ + 'BasicNT_create', + 'ExcessNT_create', + 'NTRegisteredType', + 'NTUnRegistered_create', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'cTransaction_get', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_c_struct_sequence_dir_3_11_onwards(): + result = dir(cStructSequence) + print() + print(result) + assert result == [ + 'BasicNT_create', + 'ExcessNT_create', + 'NTRegisteredType', + 'NTUnRegistered_create', + 'NTWithUnnamedField_create', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'cTransaction_get', + ] + + +def test_basic_nt_create(): + basic_nt = cStructSequence.BasicNT_create('foo', 'bar') + assert str(type(basic_nt)) == "" + + +def test_basic_nt_create_attributes(): + basic_nt = cStructSequence.BasicNT_create('foo', 'bar') + assert basic_nt.field_one == "foo" + assert basic_nt.field_two == "bar" + assert basic_nt.index("foo") == 0 + assert basic_nt.index("bar") == 1 + assert basic_nt.n_fields == 2 + assert basic_nt.n_sequence_fields == 2 + assert basic_nt.n_unnamed_fields == 0 + + +def test_nt_registered_type(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert str(type(nt)) == "" + + +def test_nt_registered_str(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert str(nt) == "cStructSequence.NTRegistered(field_one='foo', field_two='bar')" + + +def test_nt_registered_mro(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert str(type(nt).__mro__) == "(, , )" + + +@pytest.mark.skipif(not (sys.version_info.minor < 10), reason='Python < 3.10') +def test_nt_registered_dir_pre_3_10(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert dir(nt) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + # '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + # '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + # '__replace__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor == 10), reason='Python 3.10') +def test_nt_registered_dir_3_10(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert dir(nt) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor in (11, 12)), reason='Python 3.11, 3.12') +def test_nt_registered_dir_3_11_3_12(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert dir(nt) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python >= 3.13') +def test_nt_registered_dir_3_13_onwards(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert dir(nt) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__replace__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +def test_nt_registered_field_access(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert nt.field_one == 'foo' + assert nt.field_two == 'bar' + + +def test_nt_registered_index(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + assert nt[0] == 'foo' + assert nt[1] == 'bar' + assert nt[-1] == 'bar' + assert nt[-2] == 'foo' + + +def test_nt_registered_index_out_of_range(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + with pytest.raises(IndexError) as err: + value = nt[2] + assert err.value.args[0] == 'tuple index out of range' + with pytest.raises(IndexError) as err: + value = nt[-3] + assert err.value.args[0] == 'tuple index out of range' + + +def test_nt_no__make(): + with pytest.raises(AttributeError) as err: + nt = cStructSequence.NTRegisteredType._make(('foo', 'bar')) + assert err.value.args[0] == "type object 'cStructSequence.NTRegistered' has no attribute '_make'" + + +def test_nt_no__asdict(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + with pytest.raises(AttributeError) as err: + nt._asdict() + assert err.value.args[0] == "'cStructSequence.NTRegistered' object has no attribute '_asdict'" + + +def test_nt_no__replace(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + with pytest.raises(AttributeError) as err: + nt._replace(field_one='baz') + assert err.value.args[0] == "'cStructSequence.NTRegistered' object has no attribute '_replace'" + + +def test_nt_no__fields(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + with pytest.raises(AttributeError) as err: + nt._fields + assert err.value.args[0] == "'cStructSequence.NTRegistered' object has no attribute '_fields'" + + +def test_nt_no__field_defaults(): + nt = cStructSequence.NTRegisteredType(('foo', 'bar')) + with pytest.raises(AttributeError) as err: + nt._fields_defaults + assert err.value.args[0] == "'cStructSequence.NTRegistered' object has no attribute '_fields_defaults'" + + +def test_nt_unregistered_type_not_available(): + with pytest.raises(AttributeError) as err: + cStructSequence.NTUnRegistered('bar', 'foo') + assert err.value.args[0] == "module 'cPyExtPatt.cStructSequence' has no attribute 'NTUnRegistered'" + + +def test_nt_unregistered_type(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + assert str(type(ntu)) == "" + + +def test_nt_unregistered_mro(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + assert str(type(ntu).__mro__) == "(, , )" + + +@pytest.mark.skipif(not (sys.version_info.minor < 10), reason='Python < 3.10') +def test_nt_unregistered_dir_pre_3_10(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + assert dir(ntu) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + # '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + # '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + # '__replace__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor == 10), reason='Python 3.10') +def test_nt_unregistered_dir_3_10(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + assert dir(ntu) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor in (11, 12)), reason='Python 3.11, 3.12') +def test_nt_unregistered_dir_3_11_3_12(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + assert dir(ntu) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python >= 3.13') +def test_nt_unregistered_dir_3_13_onwards(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + assert dir(ntu) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__replace__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'field_two', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +def test_nt_unregistered_field_access(): + ntu = cStructSequence.NTUnRegistered_create('foo', 'bar') + assert ntu.field_one == 'foo' + assert ntu.field_two == 'bar' + + +def test_nt_unregistered_index(): + ntu = cStructSequence.NTUnRegistered_create('foo', 'bar') + assert ntu[0] == 'foo' + assert ntu[1] == 'bar' + assert ntu[-1] == 'bar' + assert ntu[-2] == 'foo' + + +def test_nt_unregistered_index_out_of_range(): + ntu = cStructSequence.NTUnRegistered_create('bar', 'foo') + with pytest.raises(IndexError) as err: + value = ntu[2] + assert err.value.args[0] == 'tuple index out of range' + with pytest.raises(IndexError) as err: + value = ntu[-3] + assert err.value.args[0] == 'tuple index out of range' + + +def test_nt_cTransaction_get_type(): + nt = cStructSequence.cTransaction_get(17145) + assert str(type(nt)) == "" + + +def test_nt_cTransaction_get_fields(): + nt = cStructSequence.cTransaction_get(17145) + assert nt.id == 17145 + assert nt.reference == "Some reference." + assert nt.amount == 42.76 + + +def test_excess_nt_create(): + nt = cStructSequence.ExcessNT_create('bar', 'foo', 'baz') + assert str(type(nt)) == "" + + +@pytest.mark.parametrize( + 'attr, value', + ( + ('n_fields', 3,), + ('n_sequence_fields', 2,), + ('n_unnamed_fields', 0,), + ) +) +def test_excess_nt_getattr(attr, value): + nt = cStructSequence.ExcessNT_create('bar', 'foo', 'baz') + result = getattr(nt, attr) + assert result == value + + +def test_excess_nt_field_three_avalible(): + nt = cStructSequence.ExcessNT_create('bar', 'foo', 'baz') + assert nt.field_three == 'baz' + + +def test_excess_nt_field_three_index_missing(): + nt = cStructSequence.ExcessNT_create('bar', 'foo', 'baz') + with pytest.raises(IndexError) as err: + nt[2] + assert err.value.args[0] == 'tuple index out of range' + + +@pytest.mark.skipif(not (11 <= sys.version_info.minor <= 12), reason='Python 3.11, 3.12') +def test_nt_with_unnamed_field_create_dir_3_11_and_before(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + print() + pprint.pprint(dir(ntuf)) + assert dir(ntuf) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 13), reason='Python 3.13+') +def test_nt_with_unnamed_field_create_dir_3_12_onwards(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + print() + pprint.pprint(dir(ntuf)) + assert dir(ntuf) == [ + '__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getnewargs__', + '__getstate__', + '__gt__', + '__hash__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__match_args__', + '__module__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__replace__', + '__repr__', + '__rmul__', + '__setattr__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'count', + 'field_one', + 'index', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields' + ] + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_len(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert len(ntuf) == 1 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_n_fields(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert ntuf.n_fields == 2 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_n_sequence_fields(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert ntuf.n_sequence_fields == 1 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_n_unnamed_fields(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert ntuf.n_unnamed_fields == 1 + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_index_tuple(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert tuple(ntuf) == ('foo',) + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_index_fields(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert ntuf[0] == 'foo' + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_index_fields_raises(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + with pytest.raises(IndexError) as err: + assert ntuf[1] == 'bar' + assert err.value.args[0] == 'tuple index out of range' + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_repr(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert repr(ntuf) == "cStructSequence.NTWithUnnamedField(field_one='foo')" + + +@pytest.mark.skipif(not (sys.version_info.minor >= 11), reason='Python 3.11+') +def test_nt_with_unnamed_field_create_str(): + ntuf = cStructSequence.NTWithUnnamedField_create('foo', 'bar', 'baz') + assert str(ntuf) == "cStructSequence.NTWithUnnamedField(field_one='foo')" diff --git a/tests/unit/test_c_subclass.py b/tests/unit/test_c_subclass.py new file mode 100644 index 0000000..0530ad7 --- /dev/null +++ b/tests/unit/test_c_subclass.py @@ -0,0 +1,149 @@ +import datetime +import sys +import zoneinfo + +import pytest + +from cPyExtPatt.SubClass import sublist + + + + +def test_sublist_dir(): + result = dir(sublist) + assert result == ['SubList', + '__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__'] + + + +@pytest.mark.skipif(not (sys.version_info.minor <= 10), reason='Python 3.9, 3.10') +def test_sublist_sublist_dir_pre_311(): + sublist_object = sublist.SubList() + result = dir(sublist_object) + assert result == ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'appends', + 'clear', + 'copy', + 'count', + 'extend', + 'increment', + 'index', + 'insert', + 'pop', + 'remove', + 'reverse', + 'sort', + 'state'] + + +@pytest.mark.skipif(not (sys.version_info.minor > 10), reason='Python 3.11+') +def test_sublist_sublist_dir_post_310(): + sublist_object = sublist.SubList() + result = dir(sublist_object) + assert result == ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getstate__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'appends', + 'clear', + 'copy', + 'count', + 'extend', + 'increment', + 'index', + 'insert', + 'pop', + 'remove', + 'reverse', + 'sort', + 'state'] + + +def test_sublist_sublist_append(): + obj = sublist.SubList() + assert obj.appends == 0 + obj.append(42) + assert obj.appends == 1 + assert obj == [42, ] + + +def test_sublist_sublist_state(): + obj = sublist.SubList() + assert obj.state == 0 + obj.increment() + assert obj.state == 1 diff --git a/tests/unit/test_c_threads.py b/tests/unit/test_c_threads.py new file mode 100644 index 0000000..da8207a --- /dev/null +++ b/tests/unit/test_c_threads.py @@ -0,0 +1,408 @@ +import datetime +import sys +import threading +import time +import zoneinfo + +import faulthandler + +faulthandler.enable() + +import pytest + +from cPyExtPatt.Threads import cppsublist +from cPyExtPatt.Threads import csublist + + +def test_cppsublist_dir(): + result = dir(cppsublist) + assert result == ['__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'cppSubList', + ] + + +def test_csublist_dir(): + result = dir(csublist) + assert result == ['__doc__', + '__file__', + '__loader__', + '__name__', + '__package__', + '__spec__', + 'cSubList', + ] + + +@pytest.mark.skipif(not (sys.version_info.minor <= 10), reason='Python 3.9, 3.10') +def test_cppsublist_cppsublist_dir_pre_311(): + sublist_object = cppsublist.cppSubList() + result = dir(sublist_object) + assert result == ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'max', + 'pop', + 'remove', + 'reverse', + 'sort'] + + +@pytest.mark.skipif(not (sys.version_info.minor > 10), reason='Python 3.11+') +def test_cppsublist_cppsublist_dir_post_310(): + sublist_object = cppsublist.cppSubList() + result = dir(sublist_object) + assert result == ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getstate__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'max', + 'pop', + 'remove', + 'reverse', + 'sort'] + + +@pytest.mark.skipif(not (sys.version_info.minor <= 10), reason='Python 3.9, 3.10') +def test_cppsublist_csublist_dir_pre_311(): + sublist_object = csublist.cSubList() + result = dir(sublist_object) + assert result == ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'max', + 'pop', + 'remove', + 'reverse', + 'sort'] + + +@pytest.mark.skipif(not (sys.version_info.minor > 10), reason='Python 3.11+') +def test_cppsublist_csublist_dir_post_310(): + sublist_object = csublist.cSubList() + result = dir(sublist_object) + assert result == ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getstate__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'max', + 'pop', + 'remove', + 'reverse', + 'sort'] + + +def test_cppsublist_cppsublist_append(): + obj = cppsublist.cppSubList() + obj.append(42) + assert obj == [42, ] + + +@pytest.mark.parametrize( + 'values, expected', + ( + ((0,), 0), + ((3, 2,), 3), + ((3, 2, 1,), 3), + ) +) +def test_cppsublist_cppsublist_max(values, expected): + obj = cppsublist.cppSubList() + for value in values: + obj.append(value) + assert obj.max() == expected + + +def cppsublist_max(obj, count): + print(f'sublist_max(): Thread name {threading.current_thread().name}', flush=True) + for _i in range(count): + print(f'sublist_max(): Thread name {threading.current_thread().name} Result: {obj.max()}', flush=True) + time.sleep(0.25) + print(f'sublist_max(): Thread name {threading.current_thread().name} DONE', flush=True) + + +def cppsublist_append(obj, count): + print(f'sublist_append(): Thread name {threading.current_thread().name}', flush=True) + for _i in range(count): + print(f'sublist_append(): Thread name {threading.current_thread().name}', flush=True) + obj.append(len(obj)) + time.sleep(0.25) + print(f'sublist_append(): Thread name {threading.current_thread().name} DONE', flush=True) + + +def test_threaded_cpp(): + print() + print('test_threaded_max() START', flush=True) + obj = cppsublist.cppSubList(range(128)) + threads = [] + for i in range(4): + threads.append( + threading.Thread(name=f'sublist_max[{i:2d}]', target=cppsublist_max, args=(obj, 2)) + ) + threads.append( + threading.Thread(name=f'sublist_append[{i:2d}]', target=cppsublist_append, args=(obj, 2)) + ) + for thread in threads: + thread.start() + print('Waiting for worker threads', flush=True) + main_thread = threading.current_thread() + for t in threading.enumerate(): + if t is not main_thread: + t.join() + print('Worker threads DONE', flush=True) + + +def test_csublist_csublist_ctor_range(): + obj = csublist.cSubList(range(128)) + assert obj == list(range(128)) + + +def test_csublist_csublist_append(): + obj = csublist.cSubList() + obj.append(42) + assert obj == [42, ] + + +@pytest.mark.parametrize( + 'values, expected', + ( + ((0,), 0), + ((3, 2,), 3), + ((3, 2, 1,), 3), + ) +) +def test_csublist_csublist_max(values, expected): + obj = csublist.cSubList() + for value in values: + obj.append(value) + assert obj.max() == expected + + +def csublist_max(obj, count): + print( + f'sublist_max(): Thread name {threading.current_thread().name}', + flush=True + ) + for _i in range(count): + print( + f'sublist_max(): Thread name {threading.current_thread().name}' + f' Result: {obj.max()}', + flush=True + ) + time.sleep(0.25) + print( + f'sublist_max(): Thread name {threading.current_thread().name} DONE', + flush=True + ) + + +def csublist_append(obj, count): + print( + f'sublist_append(): Thread name {threading.current_thread().name}', + flush=True + ) + for _i in range(count): + print( + f'sublist_append(): Thread name {threading.current_thread().name}', + flush=True + ) + obj.append(len(obj)) + time.sleep(0.25) + print( + f'sublist_append(): Thread name {threading.current_thread().name} DONE', + flush=True + ) + + +def test_threaded_c(): + print() + print('test_threaded_c() START', flush=True) + obj = csublist.cSubList(range(128)) + threads = [] + for i in range(4): + threads.append( + threading.Thread( + name=f'sublist_max[{i:2d}]', + target=csublist_max, + args=(obj, 2), + ) + ) + threads.append( + threading.Thread( + name=f'sublist_append[{i:2d}]', + target=csublist_append, + args=(obj, 2), + ) + ) + for thread in threads: + thread.start() + print('Waiting for worker threads', flush=True) + main_thread = threading.current_thread() + for t in threading.enumerate(): + if t is not main_thread: + t.join() + print('Worker threads DONE', flush=True) diff --git a/type_objects/Python_3.10.1.h b/type_objects/Python_3.10.1.h new file mode 100644 index 0000000..7fc107f --- /dev/null +++ b/type_objects/Python_3.10.1.h @@ -0,0 +1,92 @@ +// +// Created by Paul Ross on 28/06/2024. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_10_1_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_10_1_H + +struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + struct PyMethodDef *tp_methods; + struct PyMemberDef *tp_members; + struct PyGetSetDef *tp_getset; + // Strong reference on a heap type, borrowed reference on a static type + struct _typeobject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; +}; + +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_10_1_H diff --git a/type_objects/Python_3.11.1.h b/type_objects/Python_3.11.1.h new file mode 100644 index 0000000..fe5c8d3 --- /dev/null +++ b/type_objects/Python_3.11.1.h @@ -0,0 +1,92 @@ +// +// Created by Paul Ross on 28/06/2024. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_11_1_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_11_1_H + +struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + PyMethodDef *tp_methods; + PyMemberDef *tp_members; + PyGetSetDef *tp_getset; + // Strong reference on a heap type, borrowed reference on a static type + PyTypeObject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; +}; + +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_11_1_H diff --git a/type_objects/Python_3.12.1.h b/type_objects/Python_3.12.1.h new file mode 100644 index 0000000..c334081 --- /dev/null +++ b/type_objects/Python_3.12.1.h @@ -0,0 +1,96 @@ +// +// Created by Paul Ross on 28/06/2024. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_12_1_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_12_1_H + +struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + PyMethodDef *tp_methods; + PyMemberDef *tp_members; + PyGetSetDef *tp_getset; + // Strong reference on a heap type, borrowed reference on a static type + PyTypeObject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; /* no longer used */ + void *tp_subclasses; /* for static builtin types this is an index */ + PyObject *tp_weaklist; /* not used for static builtin types */ + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; + + /* bitset of which type-watchers care about this type */ + unsigned char tp_watched; +}; + + +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_12_1_H diff --git a/type_objects/Python_3.13.0b3.h b/type_objects/Python_3.13.0b3.h new file mode 100644 index 0000000..deff5a8 --- /dev/null +++ b/type_objects/Python_3.13.0b3.h @@ -0,0 +1,97 @@ +// +// Created by Paul Ross on 28/06/2024. +// + +#define PYTHONEXTENSIONSBASIC_PYTHON_3_13_0b3_H +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_13_0b3_H + +struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + PyMethodDef *tp_methods; + PyMemberDef *tp_members; + PyGetSetDef *tp_getset; + // Strong reference on a heap type, borrowed reference on a static type + PyTypeObject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; /* no longer used */ + void *tp_subclasses; /* for static builtin types this is an index */ + PyObject *tp_weaklist; /* not used for static builtin types */ + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; + + /* bitset of which type-watchers care about this type */ + unsigned char tp_watched; + uint16_t tp_versions_used; +}; + + +#endif // PYTHONEXTENSIONSBASIC_PYTHON_3_13_0b3_H diff --git a/type_objects/Python_3.6.2.h b/type_objects/Python_3.6.2.h new file mode 100644 index 0000000..883ffdc --- /dev/null +++ b/type_objects/Python_3.6.2.h @@ -0,0 +1,102 @@ +// +// Created by Paul Ross on 18/03/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_6_2_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_6_2_H + +#ifdef Py_LIMITED_API +typedef struct _typeobject PyTypeObject; /* opaque */ +#else +typedef struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + printfunc tp_print; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + struct PyMethodDef *tp_methods; + struct PyMemberDef *tp_members; + struct PyGetSetDef *tp_getset; + struct _typeobject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + +#ifdef COUNT_ALLOCS + /* these must be last and never explicitly initialized */ + Py_ssize_t tp_allocs; + Py_ssize_t tp_frees; + Py_ssize_t tp_maxalloc; + struct _typeobject *tp_prev; + struct _typeobject *tp_next; +#endif +} PyTypeObject; +#endif +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_6_2_H diff --git a/type_objects/Python_3.7.1.h b/type_objects/Python_3.7.1.h new file mode 100644 index 0000000..286fb1c --- /dev/null +++ b/type_objects/Python_3.7.1.h @@ -0,0 +1,102 @@ +// +// Created by Paul Ross on 18/03/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_7_1_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_7_1_H + +#ifdef Py_LIMITED_API +typedef struct _typeobject PyTypeObject; /* opaque */ +#else +typedef struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + printfunc tp_print; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + struct PyMethodDef *tp_methods; + struct PyMemberDef *tp_members; + struct PyGetSetDef *tp_getset; + struct _typeobject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + +#ifdef COUNT_ALLOCS + /* these must be last and never explicitly initialized */ + Py_ssize_t tp_allocs; + Py_ssize_t tp_frees; + Py_ssize_t tp_maxalloc; + struct _typeobject *tp_prev; + struct _typeobject *tp_next; +#endif +} PyTypeObject; +#endif +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_7_1_H diff --git a/type_objects/Python_3.8.3.h b/type_objects/Python_3.8.3.h new file mode 100644 index 0000000..8a14f76 --- /dev/null +++ b/type_objects/Python_3.8.3.h @@ -0,0 +1,103 @@ +// +// Created by Paul Ross on 18/03/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_8_3_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_8_3_H + +typedef struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + struct PyMethodDef *tp_methods; + struct PyMemberDef *tp_members; + struct PyGetSetDef *tp_getset; + struct _typeobject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; + + /* bpo-37250: kept for backwards compatibility in CPython 3.8 only */ + Py_DEPRECATED(3.8) int (*tp_print)(PyObject *, FILE *, int); + +#ifdef COUNT_ALLOCS + /* these must be last and never explicitly initialized */ + Py_ssize_t tp_allocs; + Py_ssize_t tp_frees; + Py_ssize_t tp_maxalloc; + struct _typeobject *tp_prev; + struct _typeobject *tp_next; +#endif +} PyTypeObject; + +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_8_3_H diff --git a/type_objects/Python_3.8.6.h b/type_objects/Python_3.8.6.h new file mode 100644 index 0000000..361ebeb --- /dev/null +++ b/type_objects/Python_3.8.6.h @@ -0,0 +1,103 @@ +// +// Created by Paul Ross on 18/03/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_8_6_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_8_6_H + +typedef struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + struct PyMethodDef *tp_methods; + struct PyMemberDef *tp_members; + struct PyGetSetDef *tp_getset; + struct _typeobject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; + + /* bpo-37250: kept for backwards compatibility in CPython 3.8 only */ + Py_DEPRECATED(3.8) int (*tp_print)(PyObject *, FILE *, int); + +#ifdef COUNT_ALLOCS + /* these must be last and never explicitly initialized */ + Py_ssize_t tp_allocs; + Py_ssize_t tp_frees; + Py_ssize_t tp_maxalloc; + struct _typeobject *tp_prev; + struct _typeobject *tp_next; +#endif +} PyTypeObject; + +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_8_6_H diff --git a/type_objects/Python_3.9.0.h b/type_objects/Python_3.9.0.h new file mode 100644 index 0000000..6a901ad --- /dev/null +++ b/type_objects/Python_3.9.0.h @@ -0,0 +1,91 @@ +// +// Created by Paul Ross on 18/03/2021. +// + +#ifndef PYTHONEXTENSIONSBASIC_PYTHON_3_9_0_H +#define PYTHONEXTENSIONSBASIC_PYTHON_3_9_0_H + +struct _typeobject { + PyObject_VAR_HEAD + const char *tp_name; /* For printing, in format "." */ + Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + + /* Methods to implement standard operations */ + + destructor tp_dealloc; + Py_ssize_t tp_vectorcall_offset; + getattrfunc tp_getattr; + setattrfunc tp_setattr; + PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) + or tp_reserved (Python 3) */ + reprfunc tp_repr; + + /* Method suites for standard classes */ + + PyNumberMethods *tp_as_number; + PySequenceMethods *tp_as_sequence; + PyMappingMethods *tp_as_mapping; + + /* More standard operations (here for binary compatibility) */ + + hashfunc tp_hash; + ternaryfunc tp_call; + reprfunc tp_str; + getattrofunc tp_getattro; + setattrofunc tp_setattro; + + /* Functions to access object as input/output buffer */ + PyBufferProcs *tp_as_buffer; + + /* Flags to define presence of optional/expanded features */ + unsigned long tp_flags; + + const char *tp_doc; /* Documentation string */ + + /* Assigned meaning in release 2.0 */ + /* call function for all accessible objects */ + traverseproc tp_traverse; + + /* delete references to contained objects */ + inquiry tp_clear; + + /* Assigned meaning in release 2.1 */ + /* rich comparisons */ + richcmpfunc tp_richcompare; + + /* weak reference enabler */ + Py_ssize_t tp_weaklistoffset; + + /* Iterators */ + getiterfunc tp_iter; + iternextfunc tp_iternext; + + /* Attribute descriptor and subclassing stuff */ + struct PyMethodDef *tp_methods; + struct PyMemberDef *tp_members; + struct PyGetSetDef *tp_getset; + struct _typeobject *tp_base; + PyObject *tp_dict; + descrgetfunc tp_descr_get; + descrsetfunc tp_descr_set; + Py_ssize_t tp_dictoffset; + initproc tp_init; + allocfunc tp_alloc; + newfunc tp_new; + freefunc tp_free; /* Low-level free-memory routine */ + inquiry tp_is_gc; /* For PyObject_IS_GC */ + PyObject *tp_bases; + PyObject *tp_mro; /* method resolution order */ + PyObject *tp_cache; + PyObject *tp_subclasses; + PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + vectorcallfunc tp_vectorcall; +}; + +#endif //PYTHONEXTENSIONSBASIC_PYTHON_3_9_0_H diff --git a/type_objects/README.md b/type_objects/README.md new file mode 100644 index 0000000..242507a --- /dev/null +++ b/type_objects/README.md @@ -0,0 +1,7 @@ +These are the the `struct _typeobject` for various vereesions of Python: + +From `Include/cpython/object.h` + +From each Python version named by version for easy comparison. + +Versioned header guards are included.