Nearly each Python application depends on third-party Python modules. When the application is distributed, maintaining all dependencies may become a real headache. Some systems have built-in tools facilitaing this task (ports collection on *BSD systems, RPM and Debian package system on Linuxes etc.), but on Windows there seems to be no other way than include all required modules into the application distribution package.
Python application packaging may be done with several tools (like McMillian Installer or py2exe), but none of them works with PEAK out-of-the-box, because of module import tweaks used in PEAK core.
This page describes the steps taken in our company to make a distribution for PEAK-based application.
All sources listed on this page may be used under the same terms as Python or PEAK.
Currently, the application has single executable script (module) run as Windows NT service. This module uses PEAK ZConfigInterpreter to load and run all components. There are no direct references to included components in the main script.
The distribution package contains a binary wrapper for the executable script, and single subdirectory lib containing all required Python modules (both source and binary) as well as the Python dll and executable. Python modules are distributed in source form. This helps to avoid problems with PEAK lazy modules and is unlike py2exe or Installer packages including all modules as binaries.
We use McMillian Installer to build the list of module dependencies and to perform initial collecting of the module files.
The installer may be downloaded from http://mcmillan-inc.com/dnld/installer_6a1.zip or http://mcmillan-inc.com/dnld/installer_6a1.tar.gz Installer versions prior to 6a1 have some problems and cannot be used.
We use a simple c program to wrap executable python modules. This program is compiled to Windows executable stub pyrun.exe. When run, it takes Python command line options attached to the stub by utility script build_exe.py and runs Py_Main() with these options. Additional options may be specified on the executable command line; these are added to the end of "command line" passed to Py_Main.
In addition to options processing, this executable wrapper sets PYTHONPATH to the lib subdirectory, so that all modules are first looked up in the package, and not in the installed Python library (if any). site processing is disabled, and Python optimization level is set to 2 (enable basic optimizations and discard docstrings). Current directory is set to the directory of the exe file.
pyrun.c source:
#include
#include
#define FATALERROR printf
#define MAGIC "ANK\014\013\012\011\0"
#define INT4 int
typedef struct _cookie {
char magic[8]; /* 'ANK\014\013\012\013\0' */
INT4 argv; /* pos (rel to start) of argument strings */
INT4 argc; /* number of arguments */
INT4 pyvers; /* python version (e.g. 23) for dll name */
} COOKIE;
int main(int argc, char **argv) {
char here[_MAX_PATH + 1];
char thisfile[_MAX_PATH + 1];
char pythonpath[_MAX_PATH + 1];
char *pp;
INT4 len, ii;
FILE *f_fp;
COOKIE f_cookie;
int myargc, f_argc;
char **myargv;
char **arg;
int rc;
HINSTANCE dll;
FARPROC Py_Main;
/* fill in thisfile */
if (!GetModuleFileNameA(NULL, thisfile, _MAX_PATH)) {
FATALERROR("System error - unable to load!");
return -1;
}
pp = thisfile+strlen(thisfile) - 4;
if (strnicmp(pp, ".exe", 4) != 0)
strcat(thisfile, ".exe");
/* fill in here (directory of thisfile) */
/* Note: GetModuleFileName returns an absolute path */
strcpy(here, thisfile);
for (pp=here+strlen(here); *pp != '\\' && pp >= here+2; --pp);
*++pp = '\0';
len = pp - here;
/* Physically open the file */
f_fp = fopen(thisfile, "rb");
if (f_fp == NULL) {
FATALERROR("Cannot open %s: %s\n", thisfile, strerror(errno));
return -1;
}
/* Seek to the Cookie at the end of the file. */
fseek(f_fp, 0, SEEK_END);
if (fseek(f_fp, -(int)sizeof(COOKIE), SEEK_END)) {
FATALERROR("Invalid executable %s\n", thisfile);
return -1;
}
/* Read the Cookie, and check its MAGIC bytes */
fread(&f_cookie, sizeof(COOKIE), 1, f_fp);
if (memcmp(f_cookie.magic, MAGIC, sizeof(MAGIC))) {
FATALERROR("Bad magic value in %s\n", thisfile);
return -1;
}
/* start filling myargv */
f_argc = f_cookie.argc;
myargc = argc + f_argc + 1; /* one extra for hard-coded options */
myargv = arg = calloc(myargc, sizeof(char*));
/* FIXME: check error */
*arg = strdup(argv[0]);
*(++arg) = "-SOO"; /* don't 'import site', run optimized w/o docstrings */
/* add frozen args */
fseek(f_fp, f_cookie.argv, SEEK_SET);
/* FIXME: check error */
for(ii=0; ii < f_argc; ii++) {
fread(&len, sizeof(INT4), 1, f_fp);
/* FIXME: check error */
pp = malloc(len +1);
pp[len] = '\0';
/* FIXME: check error */
fread(pp, len, 1, f_fp);
/* FIXME: check error */
pp[len] = '\0';
*(++arg) = pp;
};
/* close the file */
fclose(f_fp);
/* add command line arguments */
for(ii=1; ii < argc; ii++) *(++arg) = strdup(argv[ii]);
/* DEBUG */
/*
printf("ARGS:\n");
for(ii=0; ii < myargc; ii++)
printf("\t%s\n", myargv[ii]);
*/
/* set PYTHONPATH to 'lib' subdir */
sprintf(pythonpath, "%slib", here, here);
SetEnvironmentVariableA("PYTHONPATH", pythonpath);
/* enable optimizations */
/* SetEnvironmentVariableA("PYTHONOPTIMIZE", "2"); */
/* change current directory (script names may be relative) */
SetCurrentDirectory(here);
/* load python DLL and get PY_Main entry */
sprintf(pythonpath, "%slib\\python%02d.dll", here, f_cookie.pyvers);
dll = LoadLibraryEx(pythonpath, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
if(dll == NULL) {
FATALERROR("Cannot load %s (error %i)\n", pythonpath, GetLastError());
return -1;
};
Py_Main = GetProcAddress(dll, "Py_Main");
/* run */
rc = Py_Main(myargc, myargv);
/* FIXME? free myargv */
return rc;
}
/* vim: set ts=4 sts=4 sw=4 cino=(4 et : */
As i said above, there are no direct references to included components in the main script. We have to explicitely list all components for Installer to include and analyse them. Also some modules from PEAK and standard lib are not found automatically, and need to be named explicitely.
I did this by making a modulefinder hook for the package in which the main executable module resides. This hook has hiddenimports attribute which is a list of additional module import names. The list is composed by collecting all modules in the main components package (ank.BBS.Protocol) and adding some well-known names.
hooks/hook-ank.BBS.py source:
1 # McMillian Installer hooks for BBS Server application
2 #
3 # Copyright 2003 SIA "ANK"
4 #
5 # List hidden imports for Installer Analysis: BBS Protocols,
6 # some PEAK modules, utf-8 encoding etc.
7 #
8 # History (most recent first):
9 # 02-dec-2003 [als] created
10 #
11 try:
12 import os
13 except ImportError:
14 import sys
15 os = sys.modules["os"]
16 from pprint import pprint
17
18 hiddenimports = [
19 "encodings.utf_8",
20 "select",
21 "peak.config.load_zconfig",
22 "peak.naming.factories.peak_imports",
23 "peak.running.logs",
24 "peak.running.process",
25 "persistence.interfaces",
26 "ank.BBS.AL.PythonShell",
27 "ank.BBS.AL.WLProcessor",
28 "ank.BBS.MAC.TCP.SocketServer",
29 ]
30
31 _proto_path = ["ank", "BBS", "Protocol"]
32 for _file in os.listdir(os.path.join(*_proto_path)):
33 if _file.endswith(".py") and (_file != "__init__.py"):
34 hiddenimports.append(".".join(_proto_path + [_file[:-3]]))
35
36 for _mod in hiddenimports:
37 try:
38 __import__(_mod, globals(), locals(), ["__name__"])
39 except Exception, _err:
40 print "Cannot import %s: %s" % (_mod, _err)
41 hiddenimports.remove(_mod)
42
43 # vim: set sts=4 sw=4 et :]]>]]>
Below is the spec file for McMillian Installer:
1 # McMillian Installer script for BBS Server application
2 #
3 # Copyright 2003 SIA "ANK"
4 #
5 # This script does not use any of the Installer packaging features.
6 # The Installer is used to analyse script dependencies.
7 # After that, all TOCs are altered to build the distribution tree
8 # from module sources.
9 #
10 # The scripts are wrapped into pyrun.exe.
11 #
12 # PythonService.exe is added to run the server and python.exe
13 # is added to use as poor-man's python runtime.
14 #
15 # PEAK imports are tweaked to make Analysis successfull.
16 #
17 # Distutils build directory is used to analyze and collect BBS sources
18 #
19 # History (most recent first):
20 # 02-dec-2003 [als] created
21 #
22
23 SCRIPTS = ["ank\\BBS\\BBSService.py"]
24 DIST_DIR = os.path.abspath(os.path.join(os.path.dirname(BUILDPATH), "dist"))
25 print "paths", SPECPATH, BUILDPATH, DIST_DIR
26
27 # search for modules in distutils build dir first
28 sys.path.insert(0, os.path.abspath(
29 os.path.join(SPECPATH, "..", "build", "lib.win32-2.3")))
30
31 # load PEAK to avoid problems with lazy modules
32 from peak.api import binding, config, naming, running
33
34 # remove old build directory and distribution tree
35 if os.path.isdir(BUILDPATH):
36 shutil.rmtree(BUILDPATH)
37 os.mkdir(BUILDPATH)
38 if os.path.isdir(DIST_DIR):
39 shutil.rmtree(DIST_DIR)
40
41 # analyse dependencies
42 a = Analysis(
43 SCRIPTS,
44 hookspath=["installer\\hooks"],
45 )
46
47 # list required modules that are not found by Analisys
48 _addmodules = []
49 for _name in (
50 "site",
51 "peak.binding.api",
52 "peak.running.api",
53 ):
54 _addmodules.append((_name, sys.modules[_name].__file__, "PYMODULE"))
55
56 # convert collected pure module list to the list of python sources
57 _sources = TOC()
58 for (_name, _path, _typecode) in (a.pure + _addmodules):
59 #print (_name, _path, _typecode)
60 _name = ["lib"] + _name.split(".")
61 if _path[-1] in "co":
62 _path = _path[:-1]
63 if _path.endswith("__init__.py"):
64 _name.append("__init__")
65 _sources.append((os.path.join(*_name) + ".py", _path, _typecode))
66 # also list scripts as sources
67 for _name in SCRIPTS:
68 _sources.append((os.path.join("lib", _name), _name, "PYMODULE"))
69
70 # change binaries directory to "lib";
71 # for inner modules, convert module paths to fs paths
72 _binaries = TOC()
73 for (_name, _path, _typecode) in a.binaries:
74 if _typecode == "LINK":
75 continue
76 _basename, _ext = os.path.splitext(_name)
77 if _ext.lower() not in [".dll", ".pyd"]:
78 _basename = _name
79 _ext = ""
80 _basename = os.path.join(*_basename.split(".")) + _ext
81 _binaries.append((_basename, _path, _typecode))
82 # add Windows NT service runner
83 _binaries.append(("PythonService.exe",
84 config.fileNearModule("win32service", "PythonService.exe"), "BINARY"))
85 # add Python executable
86 _binaries.append(("python.exe", sys.executable, "BINARY"))
87
88 # list additional data
89 _data = TOC()
90 for _name in (
91 "ank/BBS/app_config.xml",
92 "ank/BBS/AL/component.xml",
93 "ank/BBS/MAC/TCP/component.xml",
94 "ank/BBS/Protocol/component.xml",
95 ):
96 _data.append((os.path.join("lib", _name), _name, "DATA"))
97 _data.append((os.path.join("lib", "peak", "peak.ini"),
98 config.fileNearModule("peak", "peak.ini"), "DATA"))
99
100 # build distribution tree
101 coll = COLLECT(
102 _sources,
103 _data,
104 _binaries,
105 strip=0,
106 upx=0,
107 name=DIST_DIR
108 )
109
110 _support_path = os.path.join(DIST_DIR, "support")
111 _lib_path = os.path.join(DIST_DIR, "lib")
112 _support_path_len = len(_support_path)
113 # move 'support' directory contents to 'lib'
114 for (_path, _dirs, _files) in os.walk(_support_path, topdown=False):
115 _newpath = _lib_path + _path[_support_path_len:]
116 for _name in _files:
117 os.rename(os.path.join(_path, _name), os.path.join(_newpath, _name))
118 for _name in _dirs:
119 os.rmdir(os.path.join(_path, _name))
120 os.rmdir(_support_path)
121
122 sys.path.insert(0, SPECPATH)
123 import build_exe
124 for _name in SCRIPTS:
125 _basename = os.path.splitext(os.path.basename(_name))[0]
126 _basename = os.path.join(DIST_DIR, _basename) + ".exe"
127 build_exe.run(_basename, os.path.join("lib", _name))
128
129 # vim: set filetype=python sts=4 sw=4 et :]]>]]>
------------------------------------------------------------------------------
-- alexander smishlajev - 07-dec-2003
![[WWW]](/wiki/img/moin-www.gif){kind=small}
McMillian Installer