commit 05a1aa9e2edb5ffd323edfa018cb1c63ede7978b
parent 358a374a3ac2653c019905041f28205eb477dc26
Author: Hannu Nyman <hannu.nyman@iki.fi>
Date: Thu, 16 Mar 2017 21:52:51 +0200
Merge pull request #4130 from commodo/python-package-src
python, python3: ship packages with byte-codes and derive python source packages
Diffstat:
8 files changed, 228 insertions(+), 74 deletions(-)
diff --git a/lang/python/Makefile b/lang/python/Makefile
@@ -12,7 +12,7 @@ include ./files/python-version.mk
PKG_NAME:=python
PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
-PKG_RELEASE:=4
+PKG_RELEASE:=5
PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION)
@@ -104,6 +104,8 @@ define PyBasePackage
define PyPackage/$(1)/filespec
ifneq ($(2),)
$(subst $(space),$(newline),$(foreach lib_file,$(2),+|$(lib_file)))
+ -|/usr/lib/python$(PYTHON_VERSION)/*/test
+ -|/usr/lib/python$(PYTHON_VERSION)/*/tests
endif
endef
endef
@@ -193,6 +195,7 @@ define Build/InstallDev
./files/python-package.mk \
./files/python-host.mk \
./files/python-version.mk \
+ ./files/python-package-install.sh \
$(STAGING_DIR)/mk/
$(CP) \
$(PKG_INSTALL_DIR)/usr/include/python$(PYTHON_VERSION) \
@@ -258,6 +261,7 @@ $(subst $(space),$(newline),$(foreach lib_file,$(PYTHON_LIB_FILES_DEL),-|$(lib_f
endef
define PyPackage/python-base/install
+ $(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib
$(LN) python$(PYTHON_VERSION) $(1)/usr/bin/python
$(LN) python$(PYTHON_VERSION) $(1)/usr/bin/python2
$(CP) $(PKG_INSTALL_DIR)/usr/lib/libpython$(PYTHON_VERSION).so* $(1)/usr/lib/
@@ -297,6 +301,7 @@ $(eval $(call HostBuild))
$(foreach package, $(PYTHON_PACKAGES), \
$(eval $(call PyPackage,$(package))) \
$(eval $(call BuildPackage,$(package))) \
+ $(eval $(call BuildPackage,$(package)-src)) \
)
$(eval $(call PyPackage,python-base))
@@ -308,3 +313,7 @@ $(eval $(call BuildPackage,python-pip-conf))
$(eval $(call BuildPackage,python-base))
$(eval $(call BuildPackage,python-light))
$(eval $(call BuildPackage,python))
+
+$(eval $(call BuildPackage,python-base-src))
+$(eval $(call BuildPackage,python-light-src))
+$(eval $(call BuildPackage,python-src))
diff --git a/lang/python/files/python-package-dev.mk b/lang/python/files/python-package-dev.mk
@@ -12,9 +12,9 @@ $(call Package/python/Default)
endef
define PyPackage/python-dev/install
- $(INSTALL_DIR) $(1)/usr/bin
+ $(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib
$(CP) $(PKG_INSTALL_DIR)/usr/bin/python*config $(1)/usr/bin
- $(CP) $(PKG_INSTALL_DIR)/usr/lib/libpython$(PYTHON_VERSION).so* $(1)/usr/lib
+ $(CP) $(PKG_INSTALL_DIR)/usr/lib/python$(PYTHON_VERSION)/config/libpython$(PYTHON_VERSION).a $(1)/usr/lib
endef
$(eval $(call PyBasePackage,python-dev, \
diff --git a/lang/python/files/python-package-install.sh b/lang/python/files/python-package-install.sh
@@ -0,0 +1,70 @@
+#!/bin/sh
+
+process_filespec() {
+ local src_dir="$1"
+ local dst_dir="$2"
+ local filespec="$3"
+ echo "$filespec" | (
+ IFS='|'
+ while read fop fspec fperm; do
+ local fop=`echo "$fop" | tr -d ' \t\n'`
+ if [ "$fop" = "+" ]; then
+ if [ ! -e "${src_dir}${fspec}" ]; then
+ echo "File not found '${src_dir}${fspec}'"
+ exit 1
+ fi
+ dpath=`dirname "$fspec"`
+ if [ -z "$fperm" ]; then
+ dperm=`stat -c "%a" ${src_dir}${dpath}`
+ fi
+ mkdir -p -m$dperm ${dst_dir}${dpath}
+ echo "copying: '$fspec'"
+ cp -fpR ${src_dir}${fspec} ${dst_dir}${dpath}/
+ if [ -n "$fperm" ]; then
+ chmod -R $fperm ${dst_dir}${fspec}
+ fi
+ elif [ "$fop" = "-" ]; then
+ echo "removing: '$fspec'"
+ rm -fR ${dst_dir}${fspec}
+ elif [ "$fop" = "=" ]; then
+ echo "setting permissions: '$fperm' on '$fspec'"
+ chmod -R $fperm ${dst_dir}${fspec}
+ fi
+ done
+ )
+}
+
+src_dir="$1"
+dst_dir="$2"
+python="$3"
+mode="$4"
+filespec="$5"
+
+process_filespec "$src_dir" "$dst_dir" "$filespec" || {
+ echo "process filespec error-ed"
+ exit 1
+}
+
+if [ "$mode" == "sources" ] ; then
+ # Copy only python source files
+ find $dst_dir -not -name "*\.py" | xargs rm -f
+ # Delete empty folders (if the case)
+ find $dst_dir/usr -type d | xargs rmdir &> /dev/null
+ rmdir $dst_dir/usr &> /dev/null
+ exit 0
+fi
+
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+# However the generated byte-codes were not portable.
+# So, we just stuck to un-optimized byte-codes,
+# which is still way better/faster than running
+# Python sources all the time.
+$python -m compileall -d '/' $dst_dir || {
+ echo "python -m compileall err-ed"
+ exit 1
+}
+# Delete source files and pyc [ un-optimized bytecode files ]
+# We may want to make this optimization thing configurable later, but not sure atm
+find $dst_dir -name "*\.py" | xargs rm -f
+
diff --git a/lang/python/files/python-package.mk b/lang/python/files/python-package.mk
@@ -34,6 +34,17 @@ endif
define PyPackage
+ define Package/$(1)-src
+ $(call Package/$(1))
+ TITLE+= (sources)
+ DEPENDS:=$$$$(foreach dep,$$$$(filter +python-%,$$$$(DEPENDS)),$$$$(dep)-src)
+ endef
+
+ define Package/$(1)-src/description
+ $(call Package/$(1)/description).
+ (Contains the Python sources for this package).
+ endef
+
# Add default PyPackage filespec none defined
ifndef PyPackage/$(1)/filespec
define PyPackage/$(1)/filespec
@@ -44,8 +55,8 @@ define PyPackage
ifndef PyPackage/$(1)/install
define PyPackage/$(1)/install
if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
- $(INSTALL_DIR) $$(1)/usr/bin \
- $(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/
+ $(INSTALL_DIR) $$(1)/usr/bin ; \
+ $(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
fi
endef
endif
@@ -53,38 +64,26 @@ define PyPackage
$(call shexport,PyPackage/$(1)/filespec)
define Package/$(1)/install
- find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
- @echo "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" | ( \
- IFS='|'; \
- while read fop fspec fperm; do \
- fop=`echo "$$$$$$$$fop" | tr -d ' \t\n'`; \
- if [ "$$$$$$$$fop" = "+" ]; then \
- if [ ! -e "$(PKG_INSTALL_DIR)$$$$$$$$fspec" ]; then \
- echo "File not found '$(PKG_INSTALL_DIR)$$$$$$$$fspec'"; \
- exit 1; \
- fi; \
- dpath=`dirname "$$$$$$$$fspec"`; \
- if [ -n "$$$$$$$$fperm" ]; then \
- dperm="-m$$$$$$$$fperm"; \
- else \
- dperm=`stat -c "%a" $(PKG_INSTALL_DIR)$$$$$$$$dpath`; \
- fi; \
- mkdir -p $$$$$$$$$dperm $$(1)$$$$$$$$dpath; \
- echo "copying: '$$$$$$$$fspec'"; \
- cp -fpR $(PKG_INSTALL_DIR)$$$$$$$$fspec $$(1)$$$$$$$$dpath/; \
- if [ -n "$$$$$$$$fperm" ]; then \
- chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
- fi; \
- elif [ "$$$$$$$$fop" = "-" ]; then \
- echo "removing: '$$$$$$$$fspec'"; \
- rm -fR $$(1)$$$$$$$$fspec; \
- elif [ "$$$$$$$$fop" = "=" ]; then \
- echo "setting permissions: '$$$$$$$$fperm' on '$$$$$$$$fspec'"; \
- chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
- fi; \
- done; \
- )
$(call PyPackage/$(1)/install,$$(1))
+ find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
+ if [ -e files/python-package-install.sh ] ; then \
+ $(SHELL) files/python-package-install.sh \
+ "$(PKG_INSTALL_DIR)" "$$(1)" \
+ "$(HOST_PYTHON_BIN)" "$$(2)" \
+ "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
+ elif [ -e $(STAGING_DIR)/mk/python-package-install.sh ] ; then \
+ $(SHELL) $(STAGING_DIR)/mk/python-package-install.sh \
+ "$(PKG_INSTALL_DIR)" "$$(1)" \
+ "$(HOST_PYTHON_BIN)" "$$(2)" \
+ "$$$$$$$$$$(call shvar,PyPackage/$(1)/filespec)" ; \
+ else \
+ echo "No 'python-package-install.sh' script found" ; \
+ exit 1 ; \
+ fi
+ endef
+
+ define Package/$(1)-src/install
+ $$(call Package/$(1)/install,$$(1),sources)
endef
endef
@@ -121,7 +120,7 @@ define Build/Compile/PyMod
cd $(PKG_BUILD_DIR)/$(strip $(1)), \
./setup.py $(2), \
$(3))
- find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
+ find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
endef
define PyBuild/Compile/Default
diff --git a/lang/python3/Makefile b/lang/python3/Makefile
@@ -14,7 +14,7 @@ PYTHON_VERSION:=$(PYTHON3_VERSION)
PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO)
PKG_NAME:=python3
-PKG_RELEASE:=4
+PKG_RELEASE:=5
PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO)
PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz
@@ -98,6 +98,8 @@ define Py3BasePackage
define Py3Package/$(1)/filespec
ifneq ($(2),)
$(subst $(space),$(newline),$(foreach lib_file,$(2),+|$(lib_file)))
+ -|/usr/lib/python$(PYTHON_VERSION)/*/test
+ -|/usr/lib/python$(PYTHON_VERSION)/*/tests
endif
endef
endef
@@ -187,6 +189,7 @@ define Build/InstallDev
./files/python3-package.mk \
./files/python3-host.mk \
./files/python3-version.mk \
+ ./files/python3-package-install.sh \
$(STAGING_DIR)/mk/
$(CP) \
$(PKG_INSTALL_DIR)/usr/include/python$(PYTHON_VERSION) \
@@ -286,6 +289,7 @@ $(eval $(call HostBuild))
$(foreach package, $(PYTHON3_PACKAGES), \
$(eval $(call Py3Package,$(package))) \
$(eval $(call BuildPackage,$(package))) \
+ $(eval $(call BuildPackage,$(package)-src)) \
)
$(eval $(call Py3Package,python3-base))
@@ -295,3 +299,7 @@ $(eval $(call Py3Package,python3))
$(eval $(call BuildPackage,python3-base))
$(eval $(call BuildPackage,python3-light))
$(eval $(call BuildPackage,python3))
+
+$(eval $(call BuildPackage,python3-base-src))
+$(eval $(call BuildPackage,python3-light-src))
+$(eval $(call BuildPackage,python3-src))
diff --git a/lang/python3/files/python3-package-dev.mk b/lang/python3/files/python3-package-dev.mk
@@ -12,10 +12,10 @@ $(call Package/python3/Default)
endef
define Py3Package/python3-dev/install
- $(INSTALL_DIR) $(1)/usr/bin
+ $(INSTALL_DIR) $(1)/usr/bin $(1)/usr/lib
$(CP) $(PKG_INSTALL_DIR)/usr/bin/python$(PYTHON3_VERSION)-config $(1)/usr/bin
$(LN) python$(PYTHON3_VERSION)-config $(1)/usr/bin/python3-config
- $(LN) python$(PYTHON_VERSION)/config-$(PYTHON_VERSION)/libpython$(PYTHON3_VERSION).a $(1)/usr/lib/libpython$(PYTHON3_VERSION).a
+ $(LN) python$(PYTHON_VERSION)/config-$(PYTHON_VERSION)/libpython$(PYTHON3_VERSION).a $(1)/usr/lib/
endef
$(eval $(call Py3BasePackage,python3-dev, \
diff --git a/lang/python3/files/python3-package-install.sh b/lang/python3/files/python3-package-install.sh
@@ -0,0 +1,69 @@
+#!/bin/sh
+
+process_filespec() {
+ local src_dir="$1"
+ local dst_dir="$2"
+ local filespec="$3"
+ echo "$filespec" | (
+ IFS='|'
+ while read fop fspec fperm; do
+ local fop=`echo "$fop" | tr -d ' \t\n'`
+ if [ "$fop" = "+" ]; then
+ if [ ! -e "${src_dir}${fspec}" ]; then
+ echo "File not found '${src_dir}${fspec}'"
+ exit 1
+ fi
+ dpath=`dirname "$fspec"`
+ if [ -z "$fperm" ]; then
+ dperm=`stat -c "%a" ${src_dir}${dpath}`
+ fi
+ mkdir -p -m$dperm ${dst_dir}${dpath}
+ echo "copying: '$fspec'"
+ cp -fpR ${src_dir}${fspec} ${dst_dir}${dpath}/
+ if [ -n "$fperm" ]; then
+ chmod -R $fperm ${dst_dir}${fspec}
+ fi
+ elif [ "$fop" = "-" ]; then
+ echo "removing: '$fspec'"
+ rm -fR ${dst_dir}${fspec}
+ elif [ "$fop" = "=" ]; then
+ echo "setting permissions: '$fperm' on '$fspec'"
+ chmod -R $fperm ${dst_dir}${fspec}
+ fi
+ done
+ )
+}
+
+src_dir="$1"
+dst_dir="$2"
+python="$3"
+mode="$4"
+filespec="$5"
+
+process_filespec "$src_dir" "$dst_dir" "$filespec" || {
+ echo "process filespec error-ed"
+ exit 1
+}
+
+if [ "$mode" == "sources" ] ; then
+ # Copy only python source files
+ find $dst_dir -not -name "*\.py" | xargs rm -f
+ # Delete empty folders (if the case)
+ find $dst_dir/usr -type d | xargs rmdir &> /dev/null
+ rmdir $dst_dir/usr &> /dev/null
+ exit 0
+fi
+
+# XXX [So that you won't goof as I did]
+# Note: Yes, I tried to use the -O & -OO flags here.
+# However the generated byte-codes were not portable.
+# So, we just stuck to un-optimized byte-codes,
+# which is still way better/faster than running
+# Python sources all the time.
+$python -m compileall -b -d '/' $dst_dir || {
+ echo "python -m compileall err-ed"
+ exit 1
+}
+# Delete source files and pyc [ un-optimized bytecode files ]
+# We may want to make this optimization thing configurable later, but not sure atm
+find $dst_dir -name "*\.py" | xargs rm -f
diff --git a/lang/python3/files/python3-package.mk b/lang/python3/files/python3-package.mk
@@ -34,6 +34,17 @@ endif
define Py3Package
+ define Package/$(1)-src
+ $(call Package/$(1))
+ TITLE+= (sources)
+ DEPENDS:=$$$$(foreach dep,$$$$(filter +python3-%,$$$$(DEPENDS)),$$$$(dep)-src)
+ endef
+
+ define Package/$(1)-src/description
+ $(call Package/$(1)/description).
+ (Contains the Python3 sources for this package).
+ endef
+
# Add default PyPackage filespec none defined
ifndef Py3Package/$(1)/filespec
define Py3Package/$(1)/filespec
@@ -44,8 +55,8 @@ define Py3Package
ifndef Py3Package/$(1)/install
define Py3Package/$(1)/install
if [ -d $(PKG_INSTALL_DIR)/usr/bin ]; then \
- $(INSTALL_DIR) $$(1)/usr/bin \
- $(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/
+ $(INSTALL_DIR) $$(1)/usr/bin ; \
+ $(CP) $(PKG_INSTALL_DIR)/usr/bin/* $$(1)/usr/bin/ ; \
fi
endef
endif
@@ -53,38 +64,26 @@ define Py3Package
$(call shexport,Py3Package/$(1)/filespec)
define Package/$(1)/install
- find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
- @echo "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" | ( \
- IFS='|'; \
- while read fop fspec fperm; do \
- fop=`echo "$$$$$$$$fop" | tr -d ' \t\n'`; \
- if [ "$$$$$$$$fop" = "+" ]; then \
- if [ ! -e "$(PKG_INSTALL_DIR)$$$$$$$$fspec" ]; then \
- echo "File not found '$(PKG_INSTALL_DIR)$$$$$$$$fspec'"; \
- exit 1; \
- fi; \
- dpath=`dirname "$$$$$$$$fspec"`; \
- if [ -n "$$$$$$$$fperm" ]; then \
- dperm="-m$$$$$$$$fperm"; \
- else \
- dperm=`stat -c "%a" $(PKG_INSTALL_DIR)$$$$$$$$dpath`; \
- fi; \
- mkdir -p $$$$$$$$$dperm $$(1)$$$$$$$$dpath; \
- echo "copying: '$$$$$$$$fspec'"; \
- cp -fpR $(PKG_INSTALL_DIR)$$$$$$$$fspec $$(1)$$$$$$$$dpath/; \
- if [ -n "$$$$$$$$fperm" ]; then \
- chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
- fi; \
- elif [ "$$$$$$$$fop" = "-" ]; then \
- echo "removing: '$$$$$$$$fspec'"; \
- rm -fR $$(1)$$$$$$$$fspec; \
- elif [ "$$$$$$$$fop" = "=" ]; then \
- echo "setting permissions: '$$$$$$$$fperm' on '$$$$$$$$fspec'"; \
- chmod -R $$$$$$$$fperm $$(1)$$$$$$$$fspec; \
- fi; \
- done; \
- )
$(call Py3Package/$(1)/install,$$(1))
+ find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
+ if [ -e files/python3-package-install.sh ] ; then \
+ $(SHELL) files/python3-package-install.sh \
+ "$(PKG_INSTALL_DIR)" "$$(1)" \
+ "$(HOST_PYTHON3_BIN)" "$$(2)" \
+ "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" ; \
+ elif [ -e $(STAGING_DIR)/mk/python3-package-install.sh ] ; then \
+ $(SHELL) $(STAGING_DIR)/mk/python3-package-install.sh \
+ "$(PKG_INSTALL_DIR)" "$$(1)" \
+ "$(HOST_PYTHON3_BIN)" "$$(2)" \
+ "$$$$$$$$$$(call shvar,Py3Package/$(1)/filespec)" ; \
+ else \
+ echo "No 'python3-package-install.sh' script found" ; \
+ exit 1 ; \
+ fi
+ endef
+
+ define Package/$(1)-src/install
+ $$(call Package/$(1)/install,$$(1),sources)
endef
endef
@@ -121,7 +120,7 @@ define Build/Compile/Py3Mod
cd $(PKG_BUILD_DIR)/$(strip $(1)), \
./setup.py $(2), \
$(3))
- find $(PKG_INSTALL_DIR) -name "*\.pyc" -o -name "*\.pyo" -o -name "*\.exe" | xargs rm -f
+ find $(PKG_INSTALL_DIR) -name "*\.exe" | xargs rm -f
endef
define Py3Build/Compile/Default
