Python Extensions with C Custom PyObject?

[babaliaris]

Hello!

I'm trying to learn how to create python extensions with c. I read the documentation and i tried to make an example but a problem occurred which i don't know how to solve.

The extension is a Linked list which stores PyObjects. I want the end user to use my extension like this:

import List

l = List.create()

l.append(PyObject)

l.remove(PyObject)

The Problem is this:

PyObject *list_create(PyObject *self, PyObject *args) {


	//Create The List.
	List* new_list = List_Create();


	//Raise Memory Exception.
	if (new_list == NULL)
	{
		PyErr_SetString(PyExc_MemoryError, "Memory is Full.");
		return NULL;
	}

	//PyObject Var.
	PyObject *obj = Py_BuildValue("&O", new_list);


	//Return it.
	return obj;
}

Obviously the line:

//PyObject Var.
PyObject *obj = Py_BuildValue("&O", new_list);

Is not working as i expected. I want somehow to create a PyObject that is pointing to a List * pointer and return it to the user.

The list interface is this:

#ifndef List_H
#define List_H

#include <Python.h>

typedef struct Node
{
	PyObject *value;

	struct Node *next;
	struct Node *prev;


}Node;


//List Structure.
typedef struct List
{
	Node *head;
	Node *tail;

	unsigned int count;

}List;


//Create a new List.
List *List_Create();

//Append A New PyObject to the list.
int List_Append(List * list_obj, PyObject *obj);


//Remove a PyObject from the list.
int List_Remove(List * list_obj, PyObject *obj);


//Destroy the list.
void List_Destroy(List * list_obj);

#endif // !List_H

So basically i want somehow to wrap this functionality to a PyObject. Also i want somehow when the PyObject's reference count reaches zero, List_Destroy(List * list_obj) be called automaticly so the end user won't have to do something like that:

l.Destroy()

If You Want to see the List implementation:

#include "List.h"


List *List_Create()
{
	//Create a new list object.
	List *new_list = (List *)PyMem_Malloc(sizeof(List));


	//Initialize the list object.
	if (new_list != NULL)
	{
		new_list->count = 0;
		new_list->head  = NULL;
		new_list->tail  = NULL;
	}


	//Return it.
	return new_list;
}






int List_Append(List * list_obj, PyObject *obj)
{
	/*Returns,  -1 if memory is full, 
	            -2 if the list_obj is null, 
	             0 on success.
	*/


	//Check if the list_obj is valid.
	if (list_obj != NULL)
	{
		
		//Create A new node.
		Node *new_node = (Node *)PyMem_Malloc(sizeof(Node));

		//Memory is Full.
		if (new_node == NULL)
			return -1;


		//Initialize The Node//
		new_node->next  = NULL;
		new_node->prev  = NULL;
		new_node->value = obj;
		//Initialize The Node//



		//First Node to be added in the list.
		if (list_obj->head == NULL)
		{
			list_obj->head = new_node;
			list_obj->tail = new_node;
			list_obj->count++;
		}


		//Append a new node to the list.
		else
		{
			list_obj->tail->next = new_node;
			new_node->prev       = list_obj->tail;
			list_obj->tail       = new_node;
			list_obj->count++;
		}


		//Return successfully.
		return 0;
	}


	//list_object was not initialized.
	return -2;
}




int List_Remove(List * list_obj, PyObject *obj)
{

	/*Returns,  -1 if object to be deleted was not found in the list,
				-2 if the list_obj is null or the list is empty,
				 0 on success.
	*/


	//Check if the list_obj is valid.
	if (list_obj != NULL && list_obj->head != NULL)
	{

		//Start from the head of the list.
		Node *current = list_obj->head;

		//Find the node to be deleted.
		while (current != NULL)
		{

			//Check if you found the object to be deleted.
			if (current->value == obj)
				break;


			//Move to the next node.
			current = current->next;
		}

		//Node to be deleted was not found.
		if (current == NULL)
			return -1;


		//This is the last node inside the list.
		if (current->prev == NULL && current->next == NULL)
			list_obj->head = NULL;


		//Deleting the head of the list.
		else if (current->prev == NULL && current->next != NULL)
			list_obj->head = current->next;


		//Deleting the tail of the list.
		else if (current->next == NULL)
			list_obj->tail = current->prev;


		//Deleting an inner node.
		else
			current->prev = current->next;


		//Free the memory of the current node.
		PyMem_Free(current);


		//Return successfully.
		return 0;

	}

	return -2;
}




void List_Destroy(List * list_obj)
{
	//Check if the list_obj is valid.
	if (list_obj != NULL && list_obj->head != NULL)
	{
		//Start from the head of the list.
		Node *current = list_obj->head;
		Node *temp;

		//Free All The Nodes.
		while (current != NULL)
		{
			temp    = current;
			current = current->next;
			PyMem_Free(temp);
		}

		//Set the head to NULL.
		list_obj->head = NULL;
	}
}

Does anyone know how to do that?

Thank you.

[Gribouillis]

It won't work this way. If you want to be able to call l.append(), l.remove() etc, you need to define a new python type for your lists. The way to define a new python type from C is described <here> in the documentation.

Try to code, compile and run completely the Noddy object type and module from the documentation. It will give you a good start towards your goal.

[babaliaris]

It won't work this way. If you want to be able to call l.append(), l.remove() etc, you need to define a new python type for your lists. The way to define a new python type from C is described <here> in the documentation.

Try to code, compile and run completely the Noddy object type and module from the documentation. It will give you a good start towards your goal.

Well after a lot of reading i finally understood how it works. But i have one more question.

static PyObject *List_add_item(List_Structure* self, PyObject *args, PyObject *kwds)
{

	PyObject *item;
	static char *kwlist[] = { "Item", NULL };


	//Get the object.
	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &item))
		return NULL;

	//Increase the reference of that PyObject.
	Py_XINCREF(item);

	//Append the item to the list.
	int result = List_Append(self->list, item);


	//List failed to append the item due to memory problems.
	if (result == -1)
	{
		Py_XDECREF(item);
		PyErr_SetString(PyExc_MemoryError, "Memory is full.");
		return NULL;
	}

	//There is no object to use on the list.
	else if (result == -2)
	{
		Py_XDECREF(item);
		return NULL;
	}

	Py_RETURN_NONE;
}

When you use PyArg_ParseTupleAndKeywords to get a PyObject from the interpreter, should you increase the reference of that object as i'm doing in my code?

In the documentation, it says that functions like Py_BuildValue() when they return the PyObject they also pass ownership, so you don't have to increase the reference counter, but what about PyArg_ParseTupleAndKeywords ()?

[nilamo]

I don't know, so I checked the source...

#define PyArg_ParseTupleAndKeywords _PyArg_ParseTupleAndKeywords_SizeT

Alright, so what is THAT?

int
_PyArg_ParseTupleAndKeywords_SizeT(PyObject *args,
                                  PyObject *keywords,
                                  const char *format,
                                  char **kwlist, ...)
{
    int retval;
    va_list va;

    if ((args == NULL || !PyTuple_Check(args)) ||
        (keywords != NULL && !PyDict_Check(keywords)) ||
        format == NULL ||
        kwlist == NULL)
    {
        PyErr_BadInternalCall();
        return 0;
    }

    va_start(va, kwlist);
    retval = vgetargskeywords(args, keywords, format,
                              kwlist, &va, FLAG_SIZE_T);
    va_end(va);
    return retval;
}

So no, it does no refcounting whatsoever. It only does what it says it does: it parses tuples and keywords :p

[babaliaris]

Just one more question

Is there a way to call C macros from python?

Well i'm trying to use __FILE__ and __LINE__ macros from c to my extension library. The only way i found so far is to create a method so the user will be able to call from python:


The NB_ASSERT definition is this:

//-------------------------------------Assert Macro-------------------------------------//
#define NB_ASSERT(x) \
\
	if (x);\
\
	else{\
			printf("Assertion failed:\n\tFile: %s\n\tLine: %d\n", __FILE__, __LINE__);\
		}
//-------------------------------------Assert Macro-------------------------------------//

static PyObject *NBTest_assert(PyObject* self, PyObject *args, PyObject *kwds)
{

	//Some Variables.
	PyObject   *condition;

	//Get the information from the python interpenter.
	static char *kwlist[] = { "condition", NULL };
	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &condition))
		return NULL;


	int truthy = PyObject_IsTrue(condition);

	NB_ASSERT(truthy);

	return Py_BuildValue("i", truthy);
}

Of course this does not work. When the user is calling NBTest_assert() from python, assert states the line and the file from the .C file extension and not the .py script.

Or If it is not possible, do you have any other ideas on how to solve that? I also know that python has a build in assert(), but i want to make my own because i want my assert just to state the line and the file name and not stop the program.

---

To give you more info, this is the problem that i'm trying to solve:

def unit_test():
    assert(False)

def run_test(func):
    func()

run_test( unit_test )

Assert will say:

Assertion failed at line 5 in the file test.py

But i want to say:

Assertion failed at line 2 in the file test.py

Because run_test() is calling the unit_test() function, assert assumes that the line where you called it is the line where the func() is been called.
In C you can solve this by writing a macro, but in python i have no clue.

[Gribouillis]

AFAIK PyArg_ParseTuple does not incref the item, but you don't need to incref it yourself, because there is already a reference on the item in the argument tuple. If you don't incref it, you must not decref it upon leaving the function. If your code calls List_Append, it will probably keep a reference to the item, which means that List_Append needs to incref the item one way or another.

I found a <document> that may help you.

About the assert question, can you elaborate on the issue, because I run your code and I get

Output:
Traceback (most recent call last):
  File "asserttruc.py", line 7, in <module>
    run_test( unit_test )
  File "asserttruc.py", line 5, in run_test
    func()
  File "asserttruc.py", line 2, in unit_test
    assert(False)
AssertionError

[babaliaris]

AFAIK PyArg_ParseTuple does not incref the item, but you don't need to incref it yourself, because there is already a reference on the item in the argument tuple. If you don't incref it, you must not decref it upon leaving the function. If your code calls List_Append, it will probably keep a reference to the item, which means that List_Append needs to incref the item one way or another.

I found a <document> that may help you.

About the assert question, can you elaborate on the issue, because I run your code and I get

Output:
Traceback (most recent call last):
  File "asserttruc.py", line 7, in <module>
    run_test( unit_test )
  File "asserttruc.py", line 5, in run_test
    func()
  File "asserttruc.py", line 2, in unit_test
    assert(False)
AssertionError

I want assert to do what it does without raising an exception.

Well i found this and it works.

import inspect

def ASSERT(condition):

    if (condition == False):
        callerframerecord = inspect.stack()[1]
        frame = callerframerecord[0]
        info  = inspect.getframeinfo(frame)
        print("Assertion Failed:\n")
        print ("In file    : "+info.filename)
        print ("In function: "+info.function)
        print ("At line    : "+str(info.lineno))
        

def unit_test():
    ASSERT(False)
 
def run_test(func):
    func()
 
run_test( unit_test )

Assertion Failed:

In file    : C:\Users\babaliaris\Desktop\a.py
In function: unit_test
At line    : 16

[babaliaris]

Anyway guys problem solved.

Thank you so much for you're help!!!

[nilamo]

Is there a way to call C macros from python?

No. You also can't call them from C. The preprocessor parses those, and replaces them, before the compiler ever sees it, which means the macros don't exist by the time python gets around to using the module.

[Gribouillis]

You can get a better failure report with traceback.print_stack()

import traceback
import sys

def soft_assert(cond, file=None):
    file = file or sys.stdout
    if not cond:
        print('Stack trace (most recent call last)', file=file)
        traceback.print_stack(f=sys._getframe(1), file=file)
        print('End of soft assertion failure report',file=file)


def unit_test():
    soft_assert(False)
 
def run_test(func):
    func()
 
run_test( unit_test )

My output:

Output:
Stack trace (most recent call last)
  File "asserttruc.py", line 19, in <module>
    run_test( unit_test )
  File "asserttruc.py", line 17, in run_test
    func()
  File "asserttruc.py", line 14, in unit_test
    soft_assert(False)
End of soft assertion failure report