[PEAK] Packaging peak apps
Stephen Haberman
stephenh at chase3000.com
Tue Sep 7 12:24:09 EDT 2004
> Have you tried the latest py2exe? Apparently version 0.5 uses Python
2.3's
> "import from zipfile" facility, so it seems like it should be compatible
> with PEAK's import facility (although I haven't tried it myself).
Nifty.
If that doesn't work, I had some free time when I ran into the problem so I
fixed the zip import bug in Python.
The bug is:
http://sourceforge.net/tracker/index.php?func=detail&aid=856103&group_id=547
0&atid=105470
The URL for the patch no longer works, so I've attached it to this email.
The original patch was really small, just a 2-line change to import.c to
pass a loader reference to find_module, but then it turns out the
zipimport.c was keeping around buffers or some such thing, so the patch got
larger and more complex and hence less likely to be quickly committed to
CVS.
If someone in good standing (~cough~ Phillip ~cough~) were to review, okay,
and put some weight behind the patch, it might make it in to CVS someday. I
don't know anything about the currently 2.4 release cycle, maybe it is
already too late for 2.4.
If the newest py2exe still doesn't work, you could always ship your own
patched version of Python. Ugh. But I think it would work.
- Stephen
-------------- next part --------------
cvs -z3 diff -u dist\src\Modules\zipimport.c dist\src\Lib\test\test_zipimport.py dist\src\Lib\test\test_importhooks.py dist\src\Python\import.c (in directory C:\cvs\python\)
Index: dist/src/Modules/zipimport.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Modules/zipimport.c,v
retrieving revision 1.17
diff -u -r1.17 zipimport.c
--- dist/src/Modules/zipimport.c 7 Sep 2003 13:36:48 -0000 1.17
+++ dist/src/Modules/zipimport.c 7 Sep 2004 16:13:09 -0000
@@ -5,6 +5,9 @@
#include "compile.h"
#include <time.h>
+extern time_t PyOS_GetLastModificationTime(char *, FILE *);
+ /* In getmtime.c */
+
#define IS_SOURCE 0x0
#define IS_BYTECODE 0x1
@@ -44,12 +47,14 @@
static PyTypeObject ZipImporter_Type;
static PyObject *ZipImportError;
static PyObject *zip_directory_cache = NULL;
+static PyObject *zip_mtime_cache = NULL;
/* forward decls */
-static PyObject *read_directory(char *archive);
+static int read_directory(char *archive, PyObject *files);
static PyObject *get_data(char *archive, PyObject *toc_entry);
static PyObject *get_module_code(ZipImporter *self, char *fullname,
int *p_ispackage, char **p_modpath);
+static int check_archive_mtime(char *archive, PyObject *files);
#define ZipImporter_Check(op) PyObject_TypeCheck(op, &ZipImporter_Type)
@@ -125,15 +130,19 @@
PyObject *files;
files = PyDict_GetItemString(zip_directory_cache, path);
if (files == NULL) {
- files = read_directory(buf);
- if (files == NULL)
- return -1;
- if (PyDict_SetItemString(zip_directory_cache, path,
- files) != 0)
+ files = PyDict_New();
+ if (PyDict_SetItemString(zip_directory_cache, path, files) != 0)
return -1;
}
- else
+ else {
Py_INCREF(files);
+ }
+
+ // See if the toc_entry's in files need to be recreated
+ if (check_archive_mtime(buf, files) != 0) {
+ return -1;
+ }
+
self->files = files;
}
else {
@@ -153,6 +162,8 @@
}
}
+ PySys_WriteStderr("creating a new ZipImporter for archive %s and prefix %s\n", buf, prefix);
+
self->archive = PyString_FromString(buf);
if (self->archive == NULL)
return -1;
@@ -649,10 +660,9 @@
Directories can be recognized by the trailing SEP in the name,
data_size and file_offset are 0.
*/
-static PyObject *
-read_directory(char *archive)
+static int
+read_directory(char *archive, PyObject *files)
{
- PyObject *files = NULL;
FILE *fp;
long compress, crc, data_size, file_size, file_offset, date, time;
long header_offset, name_size, header_size, header_position;
@@ -665,7 +675,7 @@
if (strlen(archive) > MAXPATHLEN) {
PyErr_SetString(PyExc_OverflowError,
"Zip path name is too long");
- return NULL;
+ return -1;
}
strcpy(path, archive);
@@ -673,7 +683,7 @@
if (fp == NULL) {
PyErr_Format(ZipImportError, "can't open Zip file: "
"'%.200s'", archive);
- return NULL;
+ return -1;
}
fseek(fp, -22, SEEK_END);
header_position = ftell(fp);
@@ -681,14 +691,14 @@
fclose(fp);
PyErr_Format(ZipImportError, "can't read Zip file: "
"'%.200s'", archive);
- return NULL;
+ return -1;
}
if (get_long((unsigned char *)endof_central_dir) != 0x06054B50) {
/* Bad: End of Central Dir signature */
fclose(fp);
PyErr_Format(ZipImportError, "not a Zip file: "
"'%.200s'", archive);
- return NULL;
+ return -1;
}
header_size = get_long((unsigned char *)endof_central_dir + 12);
@@ -696,9 +706,8 @@
arc_offset = header_position - header_offset - header_size;
header_offset += arc_offset;
- files = PyDict_New();
- if (files == NULL)
- goto error;
+ // Clear out the old toc_entries
+ PyDict_Clear(files);
length = (long)strlen(path);
path[length] = SEP;
@@ -755,11 +764,11 @@
if (Py_VerboseFlag)
PySys_WriteStderr("# zipimport: found %ld names in %s\n",
count, archive);
- return files;
+ return 0;
error:
fclose(fp);
Py_XDECREF(files);
- return NULL;
+ return -1;
}
/* Return the zlib.decompress function object, or NULL if zlib couldn't
@@ -1083,6 +1092,10 @@
if (len < 0)
return NULL;
+ if (check_archive_mtime(PyString_AsString(self->archive), self->files) != 0) {
+ return NULL;
+ }
+
for (zso = zip_searchorder; *zso->suffix; zso++) {
PyObject *code = NULL;
@@ -1120,6 +1133,50 @@
return NULL;
}
+/* Compare an archive's modification time against the one in
+ * zip_mtime_cache and have read_directory put new toc_entry's in
+ * files if the archive has changed. */
+static int
+check_archive_mtime(char *archive, PyObject *files) {
+ FILE *fp;
+ long old_mtime = -1, new_mtime = -1;
+ PyObject *temp;
+
+ PySys_WriteStderr("checking mtime of archive %s\n", archive);
+
+ fp = fopen(archive, "r");
+ if (fp == NULL) {
+ PyErr_Format(ZipImportError, "can't open Zip file: "
+ "'%.200s'", archive);
+ return -1;
+ }
+ new_mtime = PyOS_GetLastModificationTime(archive, fp);
+ fclose(fp);
+
+ temp = PyDict_GetItemString(zip_mtime_cache, archive);
+ if (temp != NULL) {
+ old_mtime = PyLong_AsLong(temp);
+ }
+
+ PySys_WriteStderr("old_mtime == %d, new_mtime == %d\n", old_mtime, new_mtime);
+
+ if (old_mtime < new_mtime) {
+ if (Py_VerboseFlag) {
+ PySys_WriteStderr("creating toc_entry cache for Zip %s\n", archive);
+ }
+ if (read_directory(archive, files) != 0) {
+ return -1;
+ }
+ PyDict_SetItemString(zip_mtime_cache, archive, PyLong_FromLong(new_mtime));
+ }
+ else {
+ if (Py_VerboseFlag) {
+ PySys_WriteStderr("not recreating toc_entry cache for Zip %s\n", archive);
+ }
+ }
+
+ return 0;
+}
/* Module init */
@@ -1132,6 +1189,8 @@
subclass of ImportError, so it can be caught as ImportError, too.\n\
- _zip_directory_cache: a dict, mapping archive paths to zip directory\n\
info dicts, as used in zipimporter._files.\n\
+- _zip_mtime_cache: a dict, mapping archive paths to the modification\n\
+ when it was last loaded.\n\
\n\
It is usually not needed to use the zipimport module explicitly; it is\n\
used by the builtin import mechanism for sys.path items that are paths\n\
@@ -1182,7 +1241,13 @@
if (zip_directory_cache == NULL)
return;
Py_INCREF(zip_directory_cache);
- if (PyModule_AddObject(mod, "_zip_directory_cache",
- zip_directory_cache) < 0)
+ if (PyModule_AddObject(mod, "_zip_directory_cache", zip_directory_cache) < 0)
+ return;
+
+ zip_mtime_cache = PyDict_New();
+ if (zip_mtime_cache == NULL)
+ return;
+ Py_INCREF(zip_mtime_cache);
+ if (PyModule_AddObject(mod, "_zip_mtime_cache", zip_mtime_cache) < 0)
return;
}
Index: dist/src/Lib/test/test_zipimport.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/test/test_zipimport.py,v
retrieving revision 1.11
diff -u -r1.11 test_zipimport.py
--- dist/src/Lib/test/test_zipimport.py 18 Nov 2003 23:00:55 -0000 1.11
+++ dist/src/Lib/test/test_zipimport.py 7 Sep 2004 16:13:09 -0000
@@ -47,6 +47,7 @@
# We're reusing the zip archive path, so we must clear the
# cached directory info.
zipimport._zip_directory_cache.clear()
+ zipimport._zip_mtime_cache.clear()
ImportHooksBaseTestCase.setUp(self)
def doTest(self, expected_ext, files, *modules, **kw):
@@ -74,6 +75,7 @@
mod = __import__(".".join(modules), globals(), locals(),
["__dummy__"])
+ sys.path.remove(TEMP_ZIP)
if expected_ext:
file = mod.get_file()
self.assertEquals(file, os.path.join(TEMP_ZIP,
@@ -208,6 +210,44 @@
self.doTest(".py", files, TESTMOD,
stuff="Some Stuff"*31)
+ def testReload(self):
+ z = ZipFile(TEMP_ZIP, "w")
+ z.compression = self.compression
+ try:
+ zinfo = ZipInfo(TESTMOD+".py", time.localtime(NOW))
+ z.writestr(zinfo, test_src)
+ z.close()
+ sys.path.insert(0, TEMP_ZIP)
+ mod = __import__(TESTMOD, globals(), locals(), ["__dummy__"])
+ self.assertEqual('ziptestmodule', mod.get_name())
+
+ # Reload once to make sure the directory toc_entry cache
+ # is not rebuilt
+ self.assertEquals(mod, reload(mod))
+ self.assertEqual('ziptestmodule', mod.get_name())
+
+ os.remove(TEMP_ZIP)
+ time.sleep(1)
+
+ # Now change the contents to make sure its recompiled
+ sys.stderr.write("doing second reload against changed archive\n")
+ modified_test_src = test_src.replace('return __name__', 'return "new"')
+ modified_test_src = modified_test_src + '\ndef get_foo(): return 1\n'
+ z = ZipFile(TEMP_ZIP, "w")
+ z.compression = self.compression
+ zinfo = ZipInfo(TESTMOD+".py", time.localtime(NOW))
+ z.writestr(zinfo, modified_test_src)
+ z.close()
+
+ # Line by Skip has to be commented out, the zip still needs to be on the path
+ # sys.path.remove(TEMP_ZIP)
+ sys.stderr.write("to the reloads\n")
+ self.assertEqual(mod, reload(mod))
+ self.assertEqual('new', mod.get_name())
+ self.assertEqual(1, mod.get_foo())
+ finally:
+ os.remove(TEMP_ZIP)
+
class CompressedZipImportTestCase(UncompressedZipImportTestCase):
compression = ZIP_DEFLATED
Index: dist/src/Lib/test/test_importhooks.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/test/test_importhooks.py,v
retrieving revision 1.2
diff -u -r1.2 test_importhooks.py
--- dist/src/Lib/test/test_importhooks.py 17 Feb 2003 14:51:41 -0000 1.2
+++ dist/src/Lib/test/test_importhooks.py 7 Sep 2004 16:13:09 -0000
@@ -145,6 +145,8 @@
class ImportHooksTestCase(ImportHooksBaseTestCase):
def doTestImports(self, importer=None):
+ global test_co
+
import hooktestmodule
import hooktestpackage
import hooktestpackage.sub
@@ -163,6 +165,22 @@
self.assertEqual(hooktestpackage.sub.__loader__, importer)
self.assertEqual(hooktestpackage.sub.subber.__loader__, importer)
+ # Test reload
+ if not importer:
+ old_test_co = test_co
+ test_co = compile(test_src + "\ndef get_foo(): return 1", "<???>", "exec")
+ TestImporter.modules = {
+ "hooktestmodule": (False, test_co),
+ "hooktestpackage": (True, test_co),
+ "hooktestpackage.sub": (True, test_co),
+ "hooktestpackage.sub.subber": (False, test_co),
+ }
+
+ reload(hooktestmodule)
+ self.assertEqual(hooktestmodule.get_name(), "hooktestmodule")
+ self.assertEqual(hooktestmodule.get_foo(), 1)
+
+
def testMetaPath(self):
i = MetaImporter()
sys.meta_path.append(i)
@@ -173,6 +191,7 @@
sys.path.append(test_path)
self.doTestImports()
+
def testBlocker(self):
mname = "exceptions" # an arbitrary harmless builtin module
if mname in sys.modules:
Index: dist/src/Python/import.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Python/import.c,v
retrieving revision 2.237
diff -u -r2.237 import.c
--- dist/src/Python/import.c 23 Aug 2004 23:37:48 -0000 2.237
+++ dist/src/Python/import.c 7 Sep 2004 16:13:10 -0000
@@ -2251,7 +2251,7 @@
PyImport_ReloadModule(PyObject *m)
{
PyObject *modules = PyImport_GetModuleDict();
- PyObject *path = NULL;
+ PyObject *path, *loader = NULL;
char *name, *subname;
char buf[MAXPATHLEN+1];
struct filedescr *fdp;
@@ -2294,11 +2294,12 @@
PyErr_Clear();
}
buf[0] = '\0';
- fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, NULL);
+ fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader);
Py_XDECREF(path);
if (fdp == NULL)
return NULL;
- newm = load_module(name, fp, buf, fdp->type, NULL);
More information about the PEAK
mailing list