# $Id$ ## @file # kBuild - Footer - Target lists - Pass 2 - Fetches. # # # Copyright (c) 2004-2017 knut st. osmundsen # # This file is part of kBuild. # # kBuild is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version source of the License, or # (at your option) any later version. # # kBuild is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with kBuild; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # # As a special exception you are granted permission to include this file, via # the kmk include directive, as you wish without this in itself causing the # resulting makefile, program or whatever to be covered by the GPL license. # This exception does not however invalidate any other reasons why the makefile, # program, whatever should not be covered the GPL. # # ## @page pg_fetches Fetching Tools, Sources and Similar. # # The targets listed in the the FETCHES target list have the following attributes: # SOURCES # INST # FETCHTOOL # FETCHFLAGS # FETCHDIR # UNPACKTOOL # UNPACKFLAGS # # As usual the target name is an alias for 'creating' the target. Other # aliases are: # pass_fetches # fetch # unfetch # download # unpack # # @remark # # This is a little bit complex because we must guarantee that if a source file # changes only sligtly we must refetch it and to a proper unpacking of it. It # is also a desire that fetched archives and unpacked files can be deleted to # save space. # # Thus, we must be able to cleanup what we've unpacked should any of the # sources be removed. We do this by maintaining a file listing the files # and directories that was unpacked. This operation is named 'unfetch'. # # We make use of the SIZE and MD5 attributes for each of the sources to # create a digest that is stored in the primary target file. Subsequent # runswill compare their digest with it to decide if a refetch is required. # When a refetch is found necessary, an 'unfetch' is performed first to # clean out old files and directores. Note even changes in source order # will cause a refetch due to the way the digest is constructed and # evaluated. # # By not depending directly on the archives (nor on any unpacked files) # but on a goal made up from the archive name, size and md5, we allow # the user to delete the archives. Naturally, this means we'll have to # check and fetch missing archives before attempting to unpack them. # # @remark # # This feature will *NOT* work correctly with vanilla GNU make becuase # it makes use of includedep to avoid too many unnecessary files. # # @todo # 0. Move the fetches out into a unit. # 1. Download corruption / continuation. # 2. It's quite possible that there is one too many indirect dependency now... # ## generates the fetch rule define def_fetch_src_fetch_rule # Indirect goal for downloading something. .PRECIOUS: $(out) $(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE) : \ | $($(target)_$(srcname)_FETCH_2_DEPORD) %$$(call MSG_FETCH_DL,$(target),$(source),$(out)) @## @todo do fancy stuff like download continuation. $$(QUIET)$$(RM) -f -- $(out) $(cmds) $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out)) # Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary. $(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD) %$$(call MSG_FETCH_CHK,$(target),$(source),$(out)) $$(QUIET)$$(RM) -f -- $$@ @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5. @## @todo do fancy stuff like download continuation. $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \ || ( $$(RM_EXT) -f $(out) \ && $$(MAKE) $(out) -f $(MAKEFILE) --no-print-directory ) $$(QUIET2)$$(APPEND) $$@ _TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size) # Just a little precaution. .NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size) endef # def_fetch_src_fetch_rule $(eval-opt-var def_fetch_src_fetch_rule) ## generates the unpack rule define def_fetch_src_unpack_rule # This is the unpack rule. it has an order-only dependency on the download check. $(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \ $($(target)_$(srcname)_UNPACK_2_DEPEND) \ | $($(target)_$(srcname)_UNPACK_2_DEPORD) \ $(archive).checked_$(md5)_$(size) \ $(dir $(out)) %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst)) $$(QUIET)$$(RM) -f -- $(out) $$(QUIET)$$(MKDIR) -p -- $(dir $(out)) @# if the source archive doesn't exist fetch it (may have been deleted to save space). $$(QUIET)test -f $(archive) \ || ( $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \ && $$(MAKE) $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory ) $(cmds) $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size)) $$(QUIET2)$$(APPEND) $(out) $(notdir $(out)) $(eval _TARGET_$(target)_UNPACKED += $(out)) _TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size) .NOTPARALLEL: $(out) endef # def_fetch_src_unpack_rule $(eval-opt-var def_fetch_src_unpack_rule) ## Processes a fetch source # define def_fetch_src #$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)') # common local srcname := $(notdir $(source)) local inst := $(firstword \ $($(target)_$(source)_INST)\ $($(target)_$(srcname)_INST)\ $($(source)_INST)\ $($(srcname)_INST)\ $($(target)_INST)\ ) ifneq ($(patsubst %/,ok,$(inst)),ok) $(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst)) endif ## @todo Install-revamp: FIXME INSTARGET_$(target)_$(srcname) := $(inst) local fetchdir := $(firstword \ $($(target)_$(source)_FETCHDIR)\ $($(target)_$(srcname)_FETCHDIR)\ $($(source)_FETCHDIR)\ $($(srcname)_FETCHDIR)\ $($(target)_FETCHDIR)\ $(FETCHDIR)\ $(PATH_TARGET)\ ) local deps := \ $($(target)_$(source)_DEPS)\ $($(target)_$(srcname)_DEPS)\ $($(source)_DEPS)\ $($(srcname)_DEPS)\ $($(target)_DEPS) local orderdeps := \ $($(target)_$(source)_ORDERDEPS)\ $($(target)_$(srcname)_ORDERDEPS)\ $($(source)_ORDERDEPS)\ $($(srcname)_ORDERDEPS)\ $($(target)_ORDERDEPS) local md5 := $(firstword \ $($(target)_$(source)_MD5)\ $($(target)_$(srcname)_MD5)\ $($(source)_MD5)\ $($(srcname)_MD5)\ $($(target)_MD5)\ ) local size := $(firstword \ $($(target)_$(source)_SIZE)\ $($(target)_$(srcname)_SIZE)\ $($(source)_SIZE)\ $($(srcname)_SIZE)\ $($(target)_SIZE)\ ) clean_files += \ $($(target)_$(source)_CLEAN)\ $($(target)_$(srcname)_CLEAN)\ $($(source)_CLEAN)\ $($(srcname)_CLEAN) local dep := # not legal for fetch and unpack tools # # The fetching. # local out := $(fetchdir)/$(srcname) local archive := $(out) $(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname)) $(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname)) local dirdep := $(call DIRDEP,$(fetchdir)) local tool := $(firstword \ $($(target)_$(source)_FETCHTOOL)\ $($(target)_$(srcname)_FETCHTOOL)\ $($(target)_$(source)_TOOL)\ $($(target)_$(srcname)_TOOL)\ $($(source)_FETCHTOOL)\ $($(srcname)_FETCHTOOL)\ $($(source)_TOOL)\ $($(srcname)_TOOL)\ $($(target)_FETCHTOOL)\ $($(target)_TOOL)\ ) local flags :=\ $(TOOL_$(tool)_FETCHFLAGS)\ $(FETCHFLAGS)\ $($(target)_FETCHFLAGS)\ $($(srcname)_FETCHFLAGS)\ $($(source)_FETCHFLAGS)\ $($(target)_$(srcname)_FETCHFLAGS)\ $($(target)_$(source)_FETCHFLAGS) #$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size)) ifndef TOOL_$(tool)_FETCH_CMDS $(warning kBuild: tools: \ 1 $($(target)_$(source)_FETCHTOOL)\ 2 $($(target)_$(srcname)_FETCHTOOL)\ 3 $($(target)_$(source)_TOOL)\ 4 $($(target)_$(srcname)_TOOL)\ 5 $($(source)_FETCHTOOL)\ 6 $($(srcname)_FETCHTOOL)\ 7 $($(source)_TOOL)\ 8 $($(srcname)_TOOL)\ 9 $($(target)_FETCHTOOL)\ 10 $($(target)_TOOL) ) $(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) ) endif # call the tool local cmds := $(TOOL_$(tool)_FETCH_CMDS) $(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT) $(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE) $(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps) $(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps) # generate the fetch rule. $(eval $(def_fetch_src_fetch_rule)) # # The unpacking / installing. # local out := $(inst)_kBuild_$(target)_$(srcname)_unpacked.lst local dirdep := $(call DIRDEP,$(inst)) local tool := $(firstword \ $($(target)_$(source)_UNPACKTOOL)\ $($(target)_$(srcname)_UNPACKTOOL)\ $($(target)_$(source)_TOOL)\ $($(target)_$(srcname)_TOOL)\ $($(source)_UNPACKTOOL)\ $($(srcname)_UNPACKTOOL)\ $($(source)_TOOL)\ $($(srcname)_TOOL)\ $($(target)_UNPACKTOOL)\ $($(target)_TOOL) \ ) ifeq ($(tool),) local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname))))) $(evalval def_tools_include) endif local flags :=\ $(TOOL_$(tool)_UNPACKFLAGS)\ $(UNPACKFLAGS)\ $($(target)_UNPACKFLAGS)\ $($(srcname)_UNPACKFLAGS)\ $($(source)_UNPACKFLAGS)\ $($(target)_$(srcname)_UNPACKFLAGS)\ $($(target)_$(source)_UNPACKFLAGS) #$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size)) ifndef TOOL_$(tool)_UNPACK_CMDS $(warning kBuild: tools: \ 1 $($(target)_$(source)_UNPACKTOOL)\ 2 $($(target)_$(srcname)_UNPACKTOOL)\ 3 $($(target)_$(source)_TOOL)\ 4 $($(target)_$(srcname)_TOOL)\ 5 $($(source)_UNPACKTOOL)\ 6 $($(srcname)_UNPACKTOOL)\ 7 $($(source)_TOOL)\ 8 $($(srcname)_TOOL)\ 9 $($(target)_UNPACKTOOL)\ 10 $($(target)_TOOL) \ 11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \ ) $(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) ) endif # call the tool local cmds := $(TOOL_$(tool)_UNPACK_CMDS) $(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT) $(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE) $(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps) $(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps) # generate the fetch rule. $(eval $(def_fetch_src_unpack_rule)) _DIRS += $(inst) $(fetchdir) endef # def_fetch_src $(eval-opt-var def_fetch_src) ## # Define the target level rules for a fetch. # @param target # @param out # @param inst # @param _TARGET_$(target)_UNPACKED # @param _TARGET_$(target)_DIGEST # @param bld_trg # @param bld_trg_arch define def_fetch_rules $(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst)) %$$(call MSG_FETCH_OK,$(target)) $$(QUIET)$$(RM) -f -- $$@ $$@.tmp $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))' $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp) $$(QUIET)$$(MV) -f -- $$@.tmp $$@ $(out)_unfetched: %$$(call MSG_UNFETCH,$(target)) $$(QUIET)$$(RM) -f -- $$(addprefix $(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d')) $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \ $$(addprefix $(inst),$$(sort $$(dir $$(shell $$(CAT_EXT) $(out).lst 2> /dev/null)))) $$(QUIET)$$(RM) -f -- $(out).lst $(out) $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) $(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst)) $$(QUIET)$$(RM) -f -- $$@ %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\ ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target))) $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched if $(KBUILD_KMK_REVISION) > 2911 $$(QUIET)kmk_builtin_dircache deleted "$(dir $(out))" endif $$(QUIET)$$(if $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)) $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)") .NOTPARALLEL: $(out).lst $(out)_unfetched $(out) endef ## # Deal with one fetch target. # @param target # @param bld_trg # @param bld_trg_arch define def_fetch # common ## @todo Install-revamp: FIXME INSTARGET_$(target) := $($(target)_INST) ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok) $(error kBuild: Bad or missing INST property for target '$(target)'. \ $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST))) endif _TARGET_$(target)_FETCHED := _TARGET_$(target)_UNPACKED := _TARGET_$(target)_DIGEST := local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE)) # The 'sources'. #$ (warning dbg fetch: target=$(target) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch))) $(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\ $(evalval def_fetch_src)) # The target. local inst := $(INSTARGET_$(target)) local out := $(inst)_kBuild_fetch_$(target) $(eval includedep $(out)) $(eval $(def_fetch_rules)) # Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep). $(target): $(out) $(target)_unfetch: $(out)_unfetched _FETCHES += $(out) _DOWNLOADS += $(_TARGET_$(target)_FETCHED) _UNPACKS += $(_TARGET_$(target)_UNPACKED) _UNFETCHES += $(out)_unfetched _DIRS += $(inst) _CLEAN_FILES += $(clean_files) endef $(eval-opt-var def_fetch) # Walk the FETCH target lists. bld_trg := $(KBUILD_TARGET) bld_trg_arch := $(KBUILD_TARGET_ARCH) $(foreach target, $(_ALL_FETCHES), \ $(evalvalctx def_fetch)) # some aliases. download: $(_DOWNLOADS) unpack: $(_UNPACKS) fetch: $(_FETCHES) unfetch: $(_UNFETCHES) ifdef KBUILD_PROFILE_SELF $(evalcall def_profile_self, done fetching targets) endif