Skip to content

Commit

Permalink
[REF] win32: move to WinPython...
Browse files Browse the repository at this point in the history 
Purpose: for multiple reasons (bytecode change in python 3.6,
         missing files for some packages, py2exe is unmaintained
         ...) the py2exe solution cannot be used anymore.
         Odoo for windows will now use a portable Python that
         will be shipped with the Windows installer.
         For that purpose, a Winpython 3.6.2 is used.
         In order to build the Windows installer, a Winpython dir
         with all the Odoo requirements fullfilled, must exists on
         the build system (qemu-kvm VM).

* nsis installer:
    - Only the python dir is used to avoid packaging too much stuffs
    - better compression level
    - starts the Odoo service using nssm
    -  bump to V11
* change the default win10 build vm path
* print a warning when uncomplete addon move
* try to force remove of addon path:
  when the addons are moved to Odoo/addons, it happens that the
  destination already exists. In that case, the source addon was not
  deleted, resulting in a uneeded file duplication.
* fix version string to avoid invalid chars in windows service
* Add requirements adpated for WinPython
* package.py now shows the traceback when a build fails
* verify that a file exists before publishing
* debuild now creates an .xz file instead of .gz
* package: use logging module
* kvm CPU that works with older versions.
* fix a pexpect encoding bug
* fix the version to remove special chars '~' which is not appreciated
  by windows services

[REM] win32setup: Remove win32 python service

Purpose:
    Before this commit, the win32 service was managed by an executable
    builded from those two files with the help of py2xe.
    The win32 service is now managed by nssm which starts Odoo and therefore, those
    files are not needed anymore.
  • Loading branch information
@d-fence
d-fence committed Oct 3, 2017
1 parent 938ae83 commit 06c2346
Show file tree
Hide file tree
Showing 7 changed files with 136 additions and 167 deletions.
2 changes: 1 addition & 1 deletion odoo/release.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@
author_email = 'info@odoo.com'
license = 'LGPL-3'

nt_service_name = "odoo-server-" + series
nt_service_name = "odoo-server-" + series.replace('~','-')
108 changes: 68 additions & 40 deletions setup/package.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from __future__ import print_function
import logging
import optparse
import os
import pexpect
Expand All @@ -10,6 +11,7 @@
import subprocess
import tempfile
import time
import traceback
try:
from xmlrpc import client as xmlrpclib
except ImportError:
Expand All @@ -26,7 +28,7 @@
# Utils
#----------------------------------------------------------
exec(open(join(dirname(__file__), '..', 'odoo', 'release.py'), 'rb').read())
version = version.split('-')[0]
version = version.split('-')[0].replace('saas~','')
docker_version = version.replace('+', '')
timestamp = time.strftime("%Y%m%d", time.gmtime())
GPGPASSPHRASE = os.getenv('GPGPASSPHRASE')
Expand Down Expand Up @@ -58,7 +60,7 @@ def mkdir(d):
os.makedirs(d)

def system(l, chdir=None):
print (l)
logging.info("System call: {}".format(l))
if chdir:
cwd = os.getcwd()
os.chdir(chdir)
Expand All @@ -82,12 +84,12 @@ def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
)
if toinstallmodules:
print("Package test: FAILED. Not able to install dependencies of base.")
logging.error("Package test: FAILED. Not able to install dependencies of base.")
raise Exception("Installation of package failed")
else:
print("Package test: successfuly installed %s modules" % len(modules))
logging.info("Package test: successfuly installed %s modules" % len(modules))
else:
print("Package test: FAILED. Not able to install base.")
logging.error("Package test: FAILED. Not able to install base.")
raise Exception("Installation of package failed")

def publish(o, type, extensions):
Expand All @@ -114,8 +116,9 @@ def _publish(o, release):

published = []
for extension in extensions:
release = glob("%s/odoo_*.%s" % (o.build_dir, extension))[0]
published.append(_publish(o, release))
release = glob("%s/odoo_*.%s" % (o.build_dir, extension))
if release:
published.append(_publish(o, release[0]))
return published

class OdooDocker(object):
Expand Down Expand Up @@ -147,10 +150,10 @@ def end(self):
try:
_rpc_count_modules(port=str(self.port))
except Exception as e:
print('Exception during docker execution: %s:' % str(e))
print('Error during docker execution: printing the bash output:')
logging.error('Exception during docker execution: %s:' % str(e))
logging.error('Error during docker execution: printing the bash output:')
with open(self.log_file.name) as f:
print('\n'.join(f.readlines()))
print('\n'.join(f.readlines()), file=stderr)
raise
finally:
self.docker.close()
Expand Down Expand Up @@ -178,16 +181,16 @@ def __init__(self, o, image, ssh_key='', login='openerp'):
self.login = login

def timeout(self,signum,frame):
print("vm timeout kill",self.pid)
logging.warning("vm timeout kill",self.pid)
os.kill(self.pid,15)

def start(self):
l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -m 1024 -drive".split(" ")
l="kvm -cpu core2duo -smp 2,sockets=2,cores=1,threads=1 -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -m 1024 -drive".split(" ")
#l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
l.append('file=%s,snapshot=on'%self.image)
#l.extend(['-vnc','127.0.0.1:1'])
l.append('-nographic')
print( " ".join(l))
logging.info("Starting kvm: {}".format( " ".join(l)))
self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
time.sleep(20)
signal.alarm(2400)
Expand All @@ -203,7 +206,7 @@ def ssh(self,cmd):
l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
system(l)

def rsync(self,args,options='--delete --exclude .bzrignore'):
def rsync(self,args,options='--delete --exclude .git --exclude .tx --exclude __pycache__'):
cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
system(cmd)

Expand All @@ -213,17 +216,20 @@ def run(self):
class KVMWinBuildExe(KVM):
def run(self):
with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
f.write("VERSION=%s\n" % version)
f.write("VERSION=%s\n" % version.replace('~', '_'))
with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
with open(join(self.o.build_dir, 'setup/win32/Makefile.servicename'), 'w') as f:
f.write("SERVICENAME=%s\n" % nt_service_name)

remote_build_dir = '/cygdrive/c/odoobuild/server/'

self.ssh("mkdir -p build")
self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
self.ssh("cd build/server/setup/win32;time make allinone;")
self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
print("KVMWinBuildExe.run(): done")
logging.info("Syncing Odoo files to virtual machine...")
self.rsync('%s/ %s@127.0.0.1:%s' % (self.o.build_dir, self.login, remote_build_dir))
self.ssh("cd {}setup/win32;time make allinone;".format(remote_build_dir))
self.rsync('%s@127.0.0.1:%ssetup/win32/release/ %s/' % (self.login, remote_build_dir, self.o.build_dir), '')
logging.info("KVMWinBuildExe.run(): done")

class KVMWinTestExe(KVM):
def run(self):
Expand All @@ -242,18 +248,25 @@ def run(self):
# Stage: building
#----------------------------------------------------------
def _prepare_build_dir(o, win32=False):

cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
if not win32:
cmd += ['--exclude', 'setup/win32']
system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
try:
for addon_path in glob(join(o.build_dir, 'addons/*')):
if addon_path.split(os.path.sep)[-1] not in ADDONS_NOT_TO_PUBLISH:
for addon_path in glob(join(o.build_dir, 'addons/*')):
if addon_path.split(os.path.sep)[-1] not in ADDONS_NOT_TO_PUBLISH:
try:
shutil.move(addon_path, join(o.build_dir, 'odoo/addons'))
except shutil.Error:
# Thrown when the add-on is already in odoo/addons (if _prepare_build_dir
# has already been called once)
pass
except shutil.Error as e:
# Thrown when the add-on is already in odoo/addons (if _prepare_build_dir
# has already been called once)
logging.warning("Warning '{}' while moving addon '{}'".format(e,addon_path))
if addon_path.startswith(o.build_dir) and os.path.isdir(addon_path):
logging.info("Removing '{}'".format(addon_path))
try:
shutil.rmtree(addon_path)
except shutil.Error as rm_error:
logging.warning("Cannot remove '{}': {}".format(addon_path, rm_error))

def build_tgz(o):
system(['python3', 'setup.py', 'sdist', '--quiet', '--formats=gztar,zip'], o.build_dir)
Expand All @@ -266,7 +279,7 @@ def build_deb(o):
subprocess.call(cmd, cwd=o.build_dir)
if not o.no_debsign:
deb = pexpect.spawn('dpkg-buildpackage -rfakeroot -k%s' % GPGID, cwd=o.build_dir)
deb.logfile = stdout
deb.logfile = stdout.buffer
if GPGPASSPHRASE:
deb.expect_exact('Enter passphrase: ', timeout=1200)
deb.send(GPGPASSPHRASE + '\r\n')
Expand All @@ -292,7 +305,9 @@ def build_exe(o):
# Stage: testing
#----------------------------------------------------------
def _prepare_testing(o):
logging.info('Preparing testing')
if not o.no_tarball:
logging.info('Preparing docker container instance for tarball')
subprocess.call(["mkdir", "docker_src"], cwd=o.build_dir)
subprocess.call(["cp", "package.dfsrc", os.path.join(o.build_dir, "docker_src", "Dockerfile")],
cwd=os.path.join(o.odoo_dir, "setup"))
Expand All @@ -302,6 +317,7 @@ def _prepare_testing(o):
subprocess.call(["docker", "build", "-t", "odoo-%s-src-nightly-tests" % docker_version, "."],
cwd=os.path.join(o.build_dir, "docker_src"))
if not o.no_debian:
logging.info('Preparing docker container instance for debian')
subprocess.call(["mkdir", "docker_debian"], cwd=o.build_dir)
subprocess.call(["cp", "package.dfdebian", os.path.join(o.build_dir, "docker_debian", "Dockerfile")],
cwd=os.path.join(o.odoo_dir, "setup"))
Expand All @@ -311,13 +327,15 @@ def _prepare_testing(o):
subprocess.call(["docker", "build", "-t", "odoo-%s-debian-nightly-tests" % docker_version, "."],
cwd=os.path.join(o.build_dir, "docker_debian"))
if not o.no_rpm:
logging.info('Preparing docker container instance for RPM')
subprocess.call(["mkdir", "docker_fedora"], cwd=o.build_dir)
subprocess.call(["cp", "package.dffedora", os.path.join(o.build_dir, "docker_fedora", "Dockerfile")],
cwd=os.path.join(o.odoo_dir, "setup"))
subprocess.call(["docker", "build", "-t", "odoo-%s-fedora-nightly-tests" % docker_version, "."],
cwd=os.path.join(o.build_dir, "docker_fedora"))

def test_tgz(o):
logging.info('Testing tarball in docker')
with docker('odoo-%s-src-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy:
wheezy.release = '*.tar.gz'
wheezy.system("service postgresql start")
Expand All @@ -331,6 +349,7 @@ def test_tgz(o):
wheezy.system('su odoo -s /bin/bash -c "odoo --addons-path=/usr/local/lib/python2.7/dist-packages/odoo/addons -d mycompany &"')

def test_deb(o):
logging.info('Testing deb package in docker')
with docker('odoo-%s-debian-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy:
wheezy.release = '*.deb'
wheezy.system("service postgresql start")
Expand All @@ -341,6 +360,7 @@ def test_deb(o):
wheezy.system('su odoo -s /bin/bash -c "odoo -c /etc/odoo/odoo.conf -d mycompany &"')

def test_rpm(o):
logging.info('Testing rpm in docker')
with docker('odoo-%s-fedora-nightly-tests' % docker_version, o.build_dir, o.pub) as fedora24:
fedora24.release = '*.noarch.rpm'
# Start postgresql
Expand All @@ -353,6 +373,7 @@ def test_rpm(o):
fedora24.system('su odoo -s /bin/bash -c "odoo -c /etc/odoo/odoo.conf -d mycompany &"')

def test_exe(o):
logging.info('Testng windows installer in KVM')
KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()

#---------------------------------------------------------
Expand All @@ -379,7 +400,7 @@ def _gen_file(o, command, file_name, path):
]
# Generate files
for command in commands:
_gen_file(o, *command, temp_path)
_gen_file(o, command[0], command[-1], temp_path)
# Remove temp directory
shutil.rmtree(temp_path)

Expand Down Expand Up @@ -420,6 +441,8 @@ def options():
root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
build_dir = "%s-%s" % (root, timestamp)

log_levels = { "debug" : logging.DEBUG, "info": logging.INFO, "warning": logging.WARN, "error": logging.ERROR, "critical": logging.CRITICAL }

op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
Expand All @@ -431,13 +454,16 @@ def options():
op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")

# Windows VM
op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/win1035/win10_py35.qcow2', help="%default")
op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/win1035/id_rsa', help="%default")
op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/win1036/win10_winpy36.qcow2', help="%default")
op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/win1036/id_rsa', help="%default")
op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
op.add_option("", "--vm-winxp-python-version", default='3.5', help="Windows Python version installed in the VM (default: %default)")
op.add_option("", "--vm-winxp-python-version", default='3.6', help="Windows Python version installed in the VM (default: %default)")

op.add_option("", "--no-remove", action="store_true", help="don't remove build dir")
op.add_option("", "--logging", action="store", type="choice", choices=list(log_levels.keys()), default="info", help="Logging level")

(o, args) = op.parse_args()
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %I:%M:%S', level=log_levels[o.logging])
# derive other options
o.odoo_dir = root
o.pkg = join(o.build_dir, 'pkg')
Expand All @@ -459,16 +485,17 @@ def main():
test_tgz(o)
published_files = publish(o, 'tarball', ['tar.gz', 'zip'])
except Exception as e:
print("Won't publish the tgz release.\n Exception: %s" % str(e))
logging.error("Won't publish the tgz release.\n Exception: %s" % str(e))
if not o.no_debian:
build_deb(o)
try:
if not o.no_testing:
test_deb(o)
published_files = publish(o, 'debian', ['deb', 'dsc', 'changes', 'tar.gz'])
published_files = publish(o, 'debian', ['deb', 'dsc', 'changes', 'tar.xz'])
gen_deb_package(o, published_files)
except Exception as e:
print("Won't publish the deb release.\n Exception: %s" % str(e))
logging.error("Won't publish the deb release.\n Exception: %s" % str(e))
traceback.print_exc()
if not o.no_rpm:
build_rpm(o)
try:
Expand All @@ -477,7 +504,7 @@ def main():
published_files = publish(o, 'redhat', ['noarch.rpm'])
gen_rpm_repo(o, published_files[0])
except Exception as e:
print("Won't publish the rpm release.\n Exception: %s" % str(e))
logging.error("Won't publish the rpm release.\n Exception: %s" % str(e))
if not o.no_windows:
_prepare_build_dir(o, win32=True)
build_exe(o)
Expand All @@ -486,19 +513,20 @@ def main():
test_exe(o)
published_files = publish(o, 'windows', ['exe'])
except Exception as e:
print("Won't publish the exe release.\n Exception: %s" % str(e))
logging.error("Won't publish the exe release.\n Exception: %s" % str(e))
except Exception as e:
print('Something bad happened ! : {}'.format(e), file=stderr)
logging.error('Something bad happened ! : {}'.format(e))
traceback.print_exc()
finally:
if o.no_remove:
print('Build dir "{}" not removed'.format(o.build_dir))
logging.info('Build dir "{}" not removed'.format(o.build_dir))
else:
shutil.rmtree(o.build_dir)
print('Build dir %s removed' % o.build_dir)
logging.info('Build dir %s removed' % o.build_dir)

if not o.no_testing:
system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
print('Remaining dockers removed')
logging.info('Remaining dockers removed')


if __name__ == '__main__':
Expand Down
6 changes: 3 additions & 3 deletions setup/win32/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ LAUNCH_PY2EXE=/cygdrive/c/python${PYTHON_VERSION}/python.exe setup.py py2exe

MAKENSIS_ARGUMENTS=/DVERSION=$(VERSION) /DSERVICENAME=${SERVICENAME}

LAUNCH_MAKENSIS=/cygdrive/c/cygwin/makensis $(MAKENSIS_ARGUMENTS) setup.nsi
LAUNCH_MAKENSIS=/cygdrive/c/tools/cygwin/makensis $(MAKENSIS_ARGUMENTS) setup.nsi

default: allinone

Expand All @@ -27,8 +27,8 @@ server_clean:
rm -rf $(SERVER_DIRECTORY)/.cyg*

allinone: server_clean
(cd $(SERVER_DIRECTORY)/setup/win32 && $(LAUNCH_PY2EXE_SERVICE))
(cd $(SERVER_DIRECTORY) && $(LAUNCH_PY2EXE))
#(cd $(SERVER_DIRECTORY)/setup/win32 && $(LAUNCH_PY2EXE_SERVICE))
#(cd $(SERVER_DIRECTORY) && $(LAUNCH_PY2EXE))
(cd $(SERVER_DIRECTORY)/setup/win32 && $(LAUNCH_MAKENSIS))
(cd $(SERVER_DIRECTORY)/setup/win32 && mkdir -p $(FILES_DIRECTORY))
(cd $(SERVER_DIRECTORY)/setup/win32 && cp openerp-*.exe $(FILES_DIRECTORY)/openerp-server-setup-$(VERSION).exe)
Loading

0 comments on commit 06c2346

Please sign in to comment.