Current File : //proc/self/root/proc/thread-self/root/usr/bin/unattended-upgrades
#!/usr/bin/python3
# Copyright (c) 2005-2018 Canonical Ltd
#
# AUTHOR:
# Michael Vogt <mvo@ubuntu.com>
# Balint Reczey <rbalint@ubuntu.com>
# This file is part of unattended-upgrades
#
# unattended-upgrades is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# unattended-upgrades is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with unattended-upgrades; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
import atexit
import copy
import datetime
import errno
import email.charset
import fcntl
import fnmatch
import gettext
try:
from gi.repository.Gio import NetworkMonitor
except ImportError:
pass
import grp
import io
import locale
import logging
import logging.handlers
import re
import os
import select
import signal
import socket
import string
import subprocess
import sys
import syslog
try:
from typing import AbstractSet, cast, DefaultDict, Dict, Iterable, List
AbstractSet # pyflakes
DefaultDict # pyflakes
Dict # pyflakes
Iterable # pyflakes
List # pyflakes
from typing import Set, Tuple, Union
Set # pyflakes
Tuple # pyflakes
Union # pyflakes
except ImportError:
pass
from collections import defaultdict, namedtuple
from datetime import date
from email.message import Message
from gettext import gettext as _
from io import StringIO
from optparse import (
OptionParser,
SUPPRESS_HELP,
)
from subprocess import (
Popen,
PIPE,
)
from textwrap import wrap
import apt
import apt_inst
import apt_pkg
import distro_info
# the reboot required flag file used by packages
REBOOT_REQUIRED_FILE = "/var/run/reboot-required"
KEPT_PACKAGES_FILE = "var/lib/unattended-upgrades/kept-back"
MAIL_BINARY = "/usr/bin/mail"
SENDMAIL_BINARY = "/usr/sbin/sendmail"
USERS = "/usr/bin/users"
# no py3 lsb_release in debian :/
DISTRO_CODENAME = subprocess.check_output(
["lsb_release", "-c", "-s"], universal_newlines=True).strip() # type: str
DISTRO_DESC = subprocess.check_output(
["lsb_release", "-d", "-s"], universal_newlines=True).strip() # type: str
DISTRO_ID = subprocess.check_output(
["lsb_release", "-i", "-s"], universal_newlines=True).strip() # type: str
# Number of days before release of devel where we enable unattended
# upgrades.
DEVEL_UNTIL_RELEASE = datetime.timedelta(days=21)
# progress information is written here
PROGRESS_LOG = "/var/run/unattended-upgrades.progress"
PID_FILE = "/var/run/unattended-upgrades.pid"
LOCK_FILE = "/var/run/unattended-upgrades.lock"
# set from the sigint signal handler
SIGNAL_STOP_REQUEST = False
# messages to be logged only once
logged_msgs = set() # type: AbstractSet[str]
NEVER_PIN = -32768
class LoggingDateTime:
"""The date/time representation for the dpkg log file timestamps"""
LOG_DATE_TIME_FMT = "%Y-%m-%d %H:%M:%S"
@classmethod
def as_string(cls):
# type: () -> str
"""Return the current date and time as LOG_DATE_TIME_FMT string"""
return datetime.datetime.now().strftime(cls.LOG_DATE_TIME_FMT)
@classmethod
def from_string(cls, logstr):
# type: (str) -> datetime.datetime
"""Take a LOG_DATE_TIME_FMT string and return datetime object"""
return datetime.datetime.strptime(logstr, cls.LOG_DATE_TIME_FMT)
class UnknownMatcherError(ValueError):
pass
class NoAllowedOriginError(ValueError):
pass
PkgPin = namedtuple('PkgPin', ['pkg', 'priority'])
PkgFilePin = namedtuple('PkgFilePin', ['id', 'priority'])
class UnattendedUpgradesCache(apt.Cache):
def __init__(self, rootdir):
# type: (str) -> None
self._cached_candidate_pkgnames = set() # type: Set[str]
self.allowed_origins = get_allowed_origins()
logging.info(_("Allowed origins are: %s"),
", ".join(self.allowed_origins))
self.blacklist = apt_pkg.config.value_list(
"Unattended-Upgrade::Package-Blacklist")
logging.info(_("Initial blacklist: %s"), " ".join(self.blacklist))
self.whitelist = apt_pkg.config.value_list(
"Unattended-Upgrade::Package-Whitelist")
self.strict_whitelist = apt_pkg.config.find_b(
"Unattended-Upgrade::Package-Whitelist-Strict", False)
logging.info(_("Initial whitelist (%s): %s"),
"strict" if self.strict_whitelist else "not strict",
" ".join(self.whitelist))
apt.Cache.__init__(self, rootdir=rootdir)
# pre-heat lazy-loaded modules to avoid crash on python upgrade
datetime.datetime.strptime("", "")
# generate versioned_kernel_pkgs_regexp for later use
self.versioned_kernel_pkgs_regexp = versioned_kernel_pkgs_regexp()
self.running_kernel_pkgs_regexp = running_kernel_pkgs_regexp()
if self.versioned_kernel_pkgs_regexp:
logging.debug("Using %s regexp to find kernel packages",
self.versioned_kernel_pkgs_regexp.pattern)
else:
logging.debug("APT::VersionedKernelPackages is not set")
if self.running_kernel_pkgs_regexp:
logging.debug("Using %s regexp to find running kernel packages",
self.running_kernel_pkgs_regexp.pattern)
def find_better_version(self, pkg):
# type (apt.Package) -> apt.package.Version
if pkg.is_installed and pkg.versions[0] > pkg.installed:
logging.debug(
"Package %s has a higher version available, checking if it is "
"from an allowed origin and is not pinned down.", pkg.name)
for v in pkg.versions:
if pkg.installed < v \
and pkg.installed.policy_priority <= v.policy_priority \
and is_in_allowed_origin(v, self.allowed_origins):
return v
return None
def find_kept_packages(self, dry_run):
# type: (bool) -> KeptPkgs
""" Find kept packages not collected already """
kept_packages = KeptPkgs(set)
if dry_run:
logging.info(_("The list of kept packages can't be calculated in "
"dry-run mode."))
return kept_packages
for pkg in self:
better_version = self.find_better_version(pkg)
if better_version:
logging.info(self.kept_package_excuse(pkg._pkg,
self.blacklist,
self.whitelist,
self.strict_whitelist,
better_version))
kept_packages.add(pkg, better_version, self)
return kept_packages
def kept_package_excuse(self, pkg, # apt.Package
blacklist, # type: List[str]
whitelist, # type: List[str]
strict_whitelist, # type: bool
better_version # type: apt.package.Version
):
# type: (...) -> str
""" Log the excuse the package is kept back for """
if pkg.selected_state == apt_pkg.SELSTATE_HOLD:
return _("Package %s is marked to be held back.") % pkg.name
elif is_pkgname_in_blacklist(pkg.name, blacklist):
return _("Package %s is blacklisted.") % pkg.name
elif whitelist:
if strict_whitelist:
if not is_pkgname_in_whitelist(pkg.name, whitelist):
return (_(
"Package %s is not on the strict whitelist.")
% pkg.name)
else:
if not is_pkgname_in_whitelist(pkg.name, whitelist):
return (_(
"Package %s is not whitelisted and it is not a"
" dependency of a whitelisted package.")
% pkg.name)
elif not any([o.trusted for o in better_version.origins]):
return _("Package %s's origin is not trusted.") % pkg.name
return (_("Package %s is kept back because a related package"
" is kept back or due to local apt_preferences(5).")
% pkg.name)
def pinning_from_regex_list(self, regexps, priority):
# type: (List[str], int) -> List[PkgPin]
""" Represent blacklist as Python regexps as list of pkg pinnings"""
pins = [] # type: List[PkgPin]
for regex in regexps:
if python_regex_is_posix(regex):
pins.append(PkgPin('/^' + regex + '/', priority))
else:
# Python regex is not also an equivalent POSIX regexp.
# This is expected to be rare. Go through all the package names
# and pin all the matching ones.
for pkg in self._cache.packages:
if re.match(regex, pkg.name):
pins.append(PkgPin(pkg.name, priority))
return pins
def pinning_from_config(self):
# type: () -> List[Union[PkgPin, PkgFilePin]]
""" Represent configuration as list of pinnings
Assumes self.allowed_origins to be already set.
"""
pins = [] # type: List[Union[PkgPin, PkgFilePin]]
# mark not allowed origins with 'never' pin
for pkg_file in self._cache.file_list: # type: ignore
if not is_allowed_origin(pkg_file, self.allowed_origins):
# Set the magic 'never' pin on not allowed origins
logging.debug("Marking not allowed %s with %s pin", pkg_file,
NEVER_PIN)
pins.append(PkgFilePin(pkg_file.id, NEVER_PIN))
# TODO(rbalint) pin not trusted origins with NEVER_PIN
elif self.strict_whitelist:
# set even allowed origins to -1 and set individual package
# priorities up later
pins.append(PkgFilePin(pkg_file.id, -1))
# mark blacklisted packages with 'never' pin
pins.extend(self.pinning_from_regex_list( # type: ignore
self.blacklist, NEVER_PIN))
# set priority of whitelisted packages to high
pins.extend(self.pinning_from_regex_list( # type: ignore
self.whitelist, 900))
if self.strict_whitelist:
policy = self._depcache.policy
# pin down already pinned packages which are not on the whitelist
# to not install locally pinned up packages accidentally
for pkg in self._cache.packages:
if pkg.has_versions:
pkg_ver = policy.get_candidate_ver(pkg) # type: ignore
if pkg_ver is not None \
and policy.get_priority(pkg_ver) > -1:
# the pin is higher than set for allowed origins, thus
# there is extra pinning configuration
if not is_pkgname_in_whitelist(pkg.name,
self.whitelist):
pins.append(PkgPin(pkg.name, NEVER_PIN))
return pins
def apply_pinning(self, pins):
# type: (List[Union[PkgPin, PkgFilePin]]) -> None
""" Apply the list of pins """
policy = self._depcache.policy
pkg_files = {f.id: f for f in self._cache.file_list} # type: ignore
for pin in pins:
logging.debug("Applying pinning: %s" % str(pin))
if isinstance(pin, PkgPin):
policy.create_pin('Version', pin.pkg, '*', # type: ignore
pin.priority)
elif isinstance(pin, PkgFilePin):
logging.debug("Applying pin %s to package_file: %s"
% (pin.priority, str(pkg_files[pin.id])))
policy.set_priority(pkg_files[pin.id], # type: ignore
pin.priority)
def open(self, progress=None):
apt.Cache.open(self, progress)
# apply pinning generated from unattended-upgrades configuration
self.apply_pinning(self.pinning_from_config())
def adjust_candidate(self, pkg):
# type: (apt.Package) -> bool
""" Adjust origin and return True if adjustment took place
This is needed when e.g. a package is available in
the security pocket but there is also a package in the
updates pocket with a higher version number
"""
try:
new_cand = ver_in_allowed_origin(pkg, self.allowed_origins)
# Only adjust to lower versions to avoid flipping back and forth
# and to avoid picking a newer version, not selected by apt.
# This helps avoiding upgrades to experimental's packages.
if pkg.candidate is not None and new_cand < pkg.candidate:
logging.debug("adjusting candidate version: %s" % new_cand)
pkg.candidate = new_cand
return True
else:
return False
except NoAllowedOriginError:
return False
def call_checked(self, function, pkg, **kwargs):
""" Call function and check if package is in the wanted state
"""
try:
function(pkg, **kwargs)
except SystemError as e:
logging.warning(
_("package %s upgradable but fails to "
"be marked for upgrade (%s)"), pkg.name, e)
self.clear()
return False
return ((function == apt.package.Package.mark_upgrade
or function == apt.package.Package.mark_install)
and (pkg.marked_upgrade or pkg.marked_install))
def call_adjusted(self, function, pkg, **kwargs):
"""Call function, but with adjusting
packages in changes to come from allowed origins
Note that as a side effect more package's candidate can be
adjusted than only the one's in the final changes set.
"""
new_pkgs_to_adjust = [] # List[str]
if not is_pkg_change_allowed(pkg, self.blacklist, self.whitelist,
self.strict_whitelist):
return
if function == apt.package.Package.mark_upgrade \
and not pkg.is_upgradable:
if not apt_pkg.config.find_b("Unattended-Upgrade::Allow-downgrade",
False):
return
else:
function = apt.package.Package.mark_install
marking_succeeded = self.call_checked(function, pkg, **kwargs)
if (not marking_succeeded
or not check_changes_for_sanity(self, desired_pkg=pkg)) \
and allow_marking_fallback():
logging.debug("falling back to adjusting %s's dependencies"
% pkg.name)
self.clear()
# adjust candidates in advance if needed
for pkg_name in self._cached_candidate_pkgnames:
self.adjust_candidate(self[pkg_name])
self.adjust_candidate(pkg)
for dep in transitive_dependencies(pkg, self, level=1):
try:
self.adjust_candidate(self[dep])
except KeyError:
pass
self.call_checked(function, pkg, **kwargs)
for marked_pkg in self.get_changes():
if marked_pkg.name in self._cached_candidate_pkgnames:
continue
if not is_in_allowed_origin(marked_pkg.candidate,
self.allowed_origins):
try:
ver_in_allowed_origin(marked_pkg,
self.allowed_origins)
# important! this avoids downgrades below
if pkg.is_installed and not pkg.is_upgradable and \
apt_pkg.config.find_b("Unattended-Upgrade::Allow-"
"downgrade", False):
continue
new_pkgs_to_adjust.append(marked_pkg)
except NoAllowedOriginError:
pass
if new_pkgs_to_adjust:
new_pkg_adjusted = False
for pkg_to_adjust in new_pkgs_to_adjust:
if self.adjust_candidate(pkg_to_adjust):
self._cached_candidate_pkgnames.add(pkg_to_adjust.name)
new_pkg_adjusted = True
if new_pkg_adjusted:
self.call_adjusted(function, pkg, **kwargs)
def mark_upgrade_adjusted(self, pkg, **kwargs):
self.call_adjusted(apt.package.Package.mark_upgrade, pkg, **kwargs)
def mark_install_adjusted(self, pkg, **kwargs):
self.call_adjusted(apt.package.Package.mark_install, pkg, **kwargs)
class LogInstallProgress(apt.progress.base.InstallProgress):
""" Install progress that writes to self.progress_log
(/var/run/unattended-upgrades.progress by default)
"""
def __init__(self, logfile_dpkg, verbose=False,
progress_log="var/run/unattended-upgrades.progress"):
# type: (str, bool, str) -> None
apt.progress.base.InstallProgress.__init__(self)
self.logfile_dpkg = logfile_dpkg
self.progress_log = os.path.join(apt_pkg.config.find_dir("Dir"),
progress_log)
self.verbose = verbose
self.output_logfd = None # type: int
def status_change(self, pkg, percent, status):
# type: (str, float, str) -> None
with open(self.progress_log, "w") as f:
f.write(_("Progress: %s %% (%s)") % (percent, pkg))
def _fixup_fds(self):
# () -> None
required_fds = [0, 1, 2, # stdin, stdout, stderr
self.writefd,
self.write_stream.fileno(),
self.statusfd,
self.status_stream.fileno()
]
# ensure that our required fds close on exec
for fd in required_fds[3:]:
old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
# close all fds
proc_fd = "/proc/self/fd"
if os.path.exists(proc_fd):
error_count = 0
for fdname in os.listdir(proc_fd):
try:
fd = int(fdname)
except Exception:
print("ERROR: can not get fd for %s" % fdname)
if fd in required_fds:
continue
try:
os.close(fd)
# print("closed: ", fd)
except OSError as e:
# there will be one fd that can not be closed
# as its the fd from pythons internal diropen()
# so its ok to ignore one close error
error_count += 1
if error_count > 1:
print("ERROR: os.close(%s): %s" % (fd, e))
def _redirect_stdin(self):
# type: () -> None
REDIRECT_INPUT = os.devnull
fd = os.open(REDIRECT_INPUT, os.O_RDWR)
os.dup2(fd, 0)
def _redirect_output(self):
# type: () -> None
# do not create log in dry-run mode, just output to stdout/stderr
if not apt_pkg.config.find_b("Debug::pkgDPkgPM", False):
logfd = self._get_logfile_dpkg_fd()
os.dup2(logfd, 1)
os.dup2(logfd, 2)
def _get_logfile_dpkg_fd(self):
# type: () -> int
logfd = os.open(
self.logfile_dpkg, os.O_RDWR | os.O_APPEND | os.O_CREAT, 0o640)
try:
adm_gid = grp.getgrnam("adm").gr_gid
os.fchown(logfd, 0, adm_gid)
except (KeyError, OSError):
pass
return logfd
def update_interface(self):
# type: () -> None
# call super class first
apt.progress.base.InstallProgress.update_interface(self)
self._do_verbose_output_if_needed()
def _do_verbose_output_if_needed(self):
# type: () -> None
# if we are in debug mode, nothing to be more verbose about
if apt_pkg.config.find_b("Debug::pkgDPkgPM", False):
return
# handle verbose
if self.verbose:
if self.output_logfd is None:
self.output_logfd = os.open(self.logfile_dpkg, os.O_RDONLY)
os.lseek(self.output_logfd, 0, os.SEEK_END)
try:
select.select([self.output_logfd], [], [], 0)
# FIXME: this should be OSError, but in py2.7 it is still
# select.error
except select.error as e:
if e.errno != errno.EINTR: # type: ignore
logging.exception("select failed")
# output to stdout in verbose mode only
os.write(1, os.read(self.output_logfd, 1024))
def _log_in_dpkg_log(self, msg):
# type: (str) -> None
logfd = self._get_logfile_dpkg_fd()
os.write(logfd, msg.encode("utf-8"))
os.close(logfd)
def finish_update(self):
# type: () -> None
self._log_in_dpkg_log("Log ended: %s\n\n"
% LoggingDateTime.as_string())
def fork(self):
# type: () -> int
self._log_in_dpkg_log("Log started: %s\n"
% LoggingDateTime.as_string())
pid = os.fork()
if pid == 0:
self._fixup_fds()
self._redirect_stdin()
self._redirect_output()
return pid
class Unlocked:
"""
Context manager for unlocking the apt lock while cache.commit() is run
"""
def __enter__(self):
# type: () -> None
try:
apt_pkg.pkgsystem_unlock_inner()
except Exception:
# earlier python-apt used to leak lock
logging.warning("apt_pkg.pkgsystem_unlock() failed due to not "
"holding the lock but trying to continue")
pass
def __exit__(self, exc_type, exc_value, exc_tb):
# type: (object, object, object) -> None
apt_pkg.pkgsystem_lock_inner()
class KeptPkgs(defaultdict):
"""
Packages to keep by highest allowed pretty-printed origin
"""
def add(self, pkg, # type: apt.Package
version, # type: apt.package.Version
cache # type: UnattendedUpgradesCache
):
# type: (...) -> None
for origin in version.origins:
if is_allowed_origin(origin, cache.allowed_origins):
self[origin.origin + " " + origin.archive].add(pkg.name)
return
class UnattendedUpgradesResult:
"""
Represent the (potentially partial) results of an unattended-upgrades
run
"""
def __init__(self,
success, # type: bool
result_str="", # type: str
pkgs=[], # type: List[str]
pkgs_kept_back=KeptPkgs(set), # type: KeptPkgs
pkgs_removed=[], # type: List[str]
pkgs_kept_installed=[], # type: List[str]
update_stamp=False # type: bool
):
# type: (...) -> None
self.success = success
self.result_str = result_str
self.pkgs = pkgs
self.pkgs_kept_back = pkgs_kept_back
self.pkgs_removed = pkgs_removed
self.pkgs_kept_installed = pkgs_kept_installed
self.update_stamp = update_stamp
def is_dpkg_journal_dirty():
# type: () -> bool
"""
Return True if the dpkg journal is dirty
(similar to debSystem::CheckUpdates)
"""
d = os.path.join(
os.path.dirname(apt_pkg.config.find_file("Dir::State::status")),
"updates")
for f in os.listdir(d):
if re.match("[0-9]+", f):
return True
return False
def signal_handler(signal, frame):
# type: (int, object) -> None
logging.warning("SIGTERM received, will stop")
global SIGNAL_STOP_REQUEST
SIGNAL_STOP_REQUEST = True
def log_once(msg):
# type: (str) -> None
global logged_msgs
if msg not in logged_msgs:
logging.info(msg)
logged_msgs.add(msg) # type: ignore
def should_stop():
# type: () -> bool
"""
Return True if u-u needs to stop due to signal received or due to the
system started to run on battery.
"""
if SIGNAL_STOP_REQUEST:
logging.warning("SIGNAL received, stopping")
return True
try:
if apt_pkg.config.find_b("Unattended-Upgrade::OnlyOnACPower", True) \
and subprocess.call("on_ac_power") == 1:
logging.warning("System is on battery power, stopping")
return True
except FileNotFoundError:
log_once(
_("Checking if system is running on battery is skipped. Please "
"install powermgmt-base package to check power status and skip "
"installing updates when the system is running on battery."))
if apt_pkg.config.find_b(
"Unattended-Upgrade::Skip-Updates-On-Metered-Connections", True):
try:
if NetworkMonitor.get_network_metered(
NetworkMonitor.get_default()):
logging.warning(_("System is on metered connection, stopping"))
return True
except NameError:
log_once(_("Checking if connection is metered is skipped. Please "
"install python3-gi package to detect metered "
"connections and skip downloading updates."))
return False
def substitute(line):
# type: (str) -> str
""" substitude known mappings and return a new string
Currently supported ${distro-release}
"""
mapping = {"distro_codename": get_distro_codename(),
"distro_id": get_distro_id()}
return string.Template(line).substitute(mapping)
def get_distro_codename():
# type: () -> str
return DISTRO_CODENAME
def get_distro_id():
# type: () -> str
return DISTRO_ID
def allow_marking_fallback():
# type: () -> bool
return apt_pkg.config.find_b(
"Unattended-Upgrade::Allow-APT-Mark-Fallback",
get_distro_codename() != "sid")
def versioned_kernel_pkgs_regexp():
apt_versioned_kernel_pkgs = apt_pkg.config.value_list(
"APT::VersionedKernelPackages")
if apt_versioned_kernel_pkgs:
return re.compile("(" + "|".join(
["^" + p + "-[1-9][0-9]*\\.[0-9]+\\.[0-9]+-[0-9]+(-.+)?$"
for p in apt_versioned_kernel_pkgs]) + ")")
else:
return None
def running_kernel_pkgs_regexp():
apt_versioned_kernel_pkgs = apt_pkg.config.value_list(
"APT::VersionedKernelPackages")
if apt_versioned_kernel_pkgs:
running_kernel_version = subprocess.check_output(
["uname", "-r"], universal_newlines=True).rstrip()
kernel_escaped = re.escape(running_kernel_version)
try:
kernel_noflavor_escaped = re.escape(
re.match("[1-9][0-9]*\\.[0-9]+\\.[0-9]+-[0-9]+",
running_kernel_version)[0])
return re.compile("(" + "|".join(
[("^" + p + "-" + kernel_escaped + "$|^"
+ p + "-" + kernel_noflavor_escaped + "$")
for p in apt_versioned_kernel_pkgs]) + ")")
except TypeError:
# flavor could not be cut from version
return re.compile("(" + "|".join(
[("^" + p + "-" + kernel_escaped + "$")
for p in apt_versioned_kernel_pkgs]) + ")")
else:
return None
def get_allowed_origins_legacy():
# type: () -> List[str]
""" legacy support for old Allowed-Origins var """
allowed_origins = [] # type: List[str]
key = "Unattended-Upgrade::Allowed-Origins"
try:
for s in apt_pkg.config.value_list(key):
# if there is a ":" use that as seperator, else use spaces
if re.findall(r'(?<!\\):', s):
(distro_id, distro_codename) = re.split(r'(?<!\\):', s)
else:
(distro_id, distro_codename) = s.split()
# unescape "\:" back to ":"
distro_id = re.sub(r'\\:', ':', distro_id)
# escape "," (see LP: #824856) - can this be simpler?
distro_id = re.sub(r'([^\\]),', r'\1\\,', distro_id)
distro_codename = re.sub(r'([^\\]),', r'\1\\,', distro_codename)
# convert to new format
allowed_origins.append("o=%s,a=%s" % (substitute(distro_id),
substitute(distro_codename)))
except ValueError:
logging.error(_("Unable to parse %s." % key))
raise
return allowed_origins
def get_allowed_origins():
# type: () -> List[str]
""" return a list of allowed origins from apt.conf
This will take substitutions (like distro_id) into account.
"""
allowed_origins = get_allowed_origins_legacy()
key = "Unattended-Upgrade::Origins-Pattern"
try:
for s in apt_pkg.config.value_list(key):
allowed_origins.append(substitute(s))
except ValueError:
logging.error(_("Unable to parse %s." % key))
raise
return allowed_origins
def match_whitelist_string(whitelist, origin):
# type: (str, Union[apt.package.Origin, apt_pkg.PackageFile]) -> bool
"""
take a whitelist string in the form "origin=Debian,label=Debian-Security"
and match against the given python-apt origin. A empty whitelist string
never matches anything.
"""
whitelist = whitelist.strip()
if whitelist == "":
logging.warning("empty match string matches nothing")
return False
res = True
# make "\," the html quote equivalent
whitelist = whitelist.replace("\\,", "%2C")
for token in whitelist.split(","):
# strip and unquote the "," back
(what, value) = [s.strip().replace("%2C", ",")
for s in token.split("=")]
# logging.debug("matching %s=%s against %s" % (
# what, value, origin))
# support substitution here as well
value = substitute(value)
# first char is apt-cache policy output, send is the name
# in the Release file
if what in ("o", "origin"):
match = fnmatch.fnmatch(origin.origin, value)
elif what in ("l", "label"):
match = fnmatch.fnmatch(origin.label, value)
elif what in ("a", "suite", "archive"):
match = fnmatch.fnmatch(origin.archive, value)
elif what in ("c", "component"):
match = fnmatch.fnmatch(origin.component, value)
elif what in ("site",):
match = fnmatch.fnmatch(origin.site, value)
elif what in ("n", "codename",):
match = fnmatch.fnmatch(origin.codename, value)
else:
raise UnknownMatcherError(
"Unknown whitelist entry for matcher %s (token %s)" % (
what, token))
# update res
res = res and match
# logging.debug("matching %s=%s against %s" % (
# what, value, origin))
return res
def python_regex_is_posix(expression):
# type: (str) -> bool
""" Returns if the Python regex is also an equivalent POSIX regex """
return re.match("^[-a-zA-Z0-9\\^\\$\\+\\.:]*$", expression) is not None
def cache_commit(cache, # type: apt.Cache
logfile_dpkg, # type: str
verbose, # type: bool
iprogress=None, # type: apt.progress.base.InstallProgress
):
# type: (...) -> Tuple[bool, Exception]
"""Commit the changes from the given cache to the system"""
error = None
res = False
if iprogress is None:
iprogress = LogInstallProgress(logfile_dpkg, verbose)
try:
res = cache.commit(install_progress=iprogress)
cache.open()
except SystemError as e:
error = e
if verbose:
logging.exception("Exception happened during upgrade.")
cache.clear()
return res, error
def upgrade_normal(cache, logfile_dpkg, verbose):
# type: (apt.Cache, str, bool) -> bool
res, error = cache_commit(cache, logfile_dpkg, verbose)
if res:
logging.info(_("All upgrades installed"))
else:
logging.error(_("Installing the upgrades failed!"))
logging.error(_("error message: %s"), error)
logging.error(_("dpkg returned a error! See %s for details"),
logfile_dpkg)
return res
def upgrade_in_minimal_steps(cache, # type: UnattendedUpgradesCache
pkgs_to_upgrade, # type: List[str]
logfile_dpkg="", # type: str
verbose=False, # type: bool
):
# type: (...) -> bool
install_log = LogInstallProgress(logfile_dpkg, verbose)
res = True
# to upgrade contains the package names
to_upgrade = set(pkgs_to_upgrade)
for pkgname in upgrade_order(to_upgrade, cache):
# upgrade packages and dependencies in increasing expected size of
# package sets to upgrade/install together
if pkgname not in to_upgrade:
# pkg is upgraded in a previous set
continue
if should_stop():
return False
pkg = cache[pkgname]
try:
if pkg.is_upgradable \
or candidate_version_changed(pkg):
cache.mark_upgrade_adjusted(
pkg, from_user=not pkg.is_auto_installed)
elif not pkg.is_installed:
cache.mark_install_adjusted(pkg, from_user=False)
else:
continue
except Exception as e:
logging.warning(
_("package %s upgradable but fails to "
"be marked for upgrade (%s)"), pkgname, e)
cache.clear()
res = False
continue
# double check that we are not running into side effects like
# what could have been caused LP: #1020680
if not check_changes_for_sanity(cache):
logging.info("While building minimal partition: "
"cache has not allowed changes")
cache.clear()
continue
changes = [p.name for p in cache.get_changes()]
if not changes:
continue
# write progress log information
if len(pkgs_to_upgrade) > 0:
all_count = len(pkgs_to_upgrade)
remaining_count = all_count - len(to_upgrade)
percent = remaining_count / float(all_count * 100.0)
else:
percent = 100.0
install_log.status_change(pkg=",".join(changes),
percent=percent,
status="")
# apply changes
logging.debug("applying set %s" % changes)
res, error = cache_commit(cache, logfile_dpkg, verbose, install_log)
if error:
if verbose:
logging.exception("Exception happened during upgrade.")
logging.error(_("Installing the upgrades failed!"))
logging.error(_("error message: %s"), error)
logging.error(_("dpkg returned a error! See %s for details"),
logfile_dpkg)
return False
to_upgrade = to_upgrade - set(changes)
logging.debug("left to upgrade %s" % to_upgrade)
if len(to_upgrade) == 0:
logging.info(_("All upgrades installed"))
break
return res
def is_allowed_origin(origin, allowed_origins):
# type: (Union[apt.package.Origin, apt_pkg.PackageFile], List[str]) -> bool
# local origin is allowed by default
if origin.component == 'now' and origin.archive == 'now' and \
not origin.label and not origin.site:
return True
for allowed in allowed_origins:
if match_whitelist_string(allowed, origin):
return True
return False
def is_in_allowed_origin(ver, allowed_origins):
# type: (apt.package.Version, List[str]) -> bool
if not ver:
return False
for origin in ver.origins:
if is_allowed_origin(origin, allowed_origins):
return True
return False
def ver_in_allowed_origin(pkg, allowed_origins):
# type: (apt.Package, List[str]) -> apt.package.Version
for ver in pkg.versions:
if is_in_allowed_origin(ver, allowed_origins):
# leave as soon as we have the highest new candidate
return ver
raise NoAllowedOriginError()
def is_pkgname_in_blacklist(pkgname, blacklist):
# type: (str, List[str]) -> bool
for blacklist_regexp in blacklist:
if re.match(blacklist_regexp, pkgname):
logging.debug("skipping blacklisted package %s" % pkgname)
return True
return False
def is_pkgname_in_whitelist(pkgname, whitelist):
# type: (str, List[str]) -> bool
# a empty whitelist means the user does not want to use this feature
if not whitelist:
return True
for whitelist_regexp in whitelist:
if re.match(whitelist_regexp, pkgname):
logging.debug("only upgrading the following package %s" %
pkgname)
return True
return False
def is_pkg_change_allowed(pkg, blacklist, whitelist, strict_whitelist):
# type: (apt.Package, List[str], List[str], bool) -> bool
if is_pkgname_in_blacklist(pkg.name, blacklist):
logging.debug("pkg %s package has been blacklisted" % pkg.name)
return False
# a strict whitelist will not allow any changes not in the
# whitelist, most people will want the relaxed whitelist
# that whitelists a package but pulls in the package
# dependencies
if strict_whitelist and \
not is_pkgname_in_whitelist(pkg.name, whitelist):
logging.debug("pkg %s package is not whitelisted" %
pkg.name)
return False
if pkg._pkg.selected_state == apt_pkg.SELSTATE_HOLD:
logging.debug("pkg %s is on hold" % pkg.name)
return False
return True
def transitive_dependencies(pkg, # type: apt.Package
cache, # type: apt.Cache
acc=set(), # type AbstractSet[str]
valid_types=None, # type: AbstractSet[str]
level=None # type: int
):
# type (...) -> AbstractSet[str]
""" All (transitive) dependencies of the package
Note that alternative (|) dependencies are collected, too
"""
if not pkg.candidate or level is not None and level < 1:
return acc
for dep in pkg.candidate.dependencies:
for base_dep in dep:
if base_dep.name not in acc:
if not valid_types or base_dep.rawtype in valid_types:
acc.add(base_dep.name)
try:
transitive_dependencies(
cache[base_dep.name], cache, acc, valid_types,
level=(level - 1 if level is not None else None))
except KeyError:
pass
return acc
def upgrade_order(to_upgrade, cache):
# type: (AbstractSet[str], apt.Cache) -> List[str]
""" Sort pkg names by the expected number of other packages to be upgraded
with it. The calculation is not 100% accurate, it is an approximation.
"""
upgrade_set_sizes = {}
# calculate upgrade sets
follow_deps = {'Depends', 'PreDepends', 'Recommends'}
for pkgname in to_upgrade:
pkg = cache[pkgname]
upgrade_set_sizes[pkgname] = len(transitive_dependencies(
pkg, cache, valid_types=follow_deps).intersection(to_upgrade))
return sorted(upgrade_set_sizes, key=upgrade_set_sizes.get)
def check_changes_for_sanity(cache, desired_pkg=None):
# type: (UnattendedUpgradesCache, apt.Package) -> bool
sanity_check_result = sanity_problem(cache, desired_pkg)
if sanity_check_result is None:
return True
else:
logging.debug("sanity check failed for: %s : %s"
% (str({str(p.candidate) for p in cache.get_changes()}),
sanity_check_result))
return False
def sanity_problem(cache, desired_pkg):
# type: (UnattendedUpgradesCache, apt.Package) -> str
if cache._depcache.broken_count != 0:
return ("there are broken packages in the cache")
# If there are no packages to be installed they were kept back
if cache.install_count == 0:
return ("no package is selected to be upgraded or installed")
changes = cache.get_changes()
for pkg in changes:
if pkg.marked_delete:
return ("pkg %s is marked to be deleted" % pkg.name)
if pkg.marked_install or pkg.marked_upgrade:
# apt will never fallback from a trusted to a untrusted
# origin so its good enough if we have a single trusted one
if not any([o.trusted for o in pkg.candidate.origins]):
return ("pkg %s is not from a trusted origin" % pkg.name)
if not is_in_allowed_origin(pkg.candidate, cache.allowed_origins):
return ("pkg %s is not in an allowed origin" % pkg.name)
if not is_pkg_change_allowed(pkg,
cache.blacklist,
cache.whitelist,
cache.strict_whitelist):
return ("pkg %s is blacklisted or is not whitelisted"
% pkg.name)
# check if the package is unsafe to upgrade unattended
ignore_require_restart = apt_pkg.config.find_b(
"Unattended-Upgrade::IgnoreAppsRequireRestart", False)
upgrade_requires = pkg.candidate.record.get("Upgrade-Requires")
if pkg.marked_upgrade and ignore_require_restart is False \
and upgrade_requires == "app-restart":
return ("pkg %s requires app-restart, it is not safe to "
"upgrade it unattended")
# check that the package we want to upgrade is in the change set
if desired_pkg and desired_pkg not in changes:
return ("pkg %s to be marked for upgrade/install is not marked "
"accordingly" % desired_pkg.name)
return None
def is_deb(file):
# type: (str) -> bool
if file.endswith(".deb"):
return True
else:
return False
def pkgname_from_deb(debfile):
# type: (str) -> str
# FIXME: add error checking here
try:
control = apt_inst.DebFile(debfile).control.extractdata("control")
sections = apt_pkg.TagSection(control)
return sections["Package"]
except (IOError, SystemError) as e:
logging.error("failed to read deb file %s (%s)" % (debfile, e))
# dumb fallback
return debfile.split("_")[0]
def get_md5sum_for_file_in_deb(deb_file, conf_file):
# type: (str, str) -> str
dpkg_cmd = ["dpkg-deb", "--fsys-tarfile", deb_file]
tar_cmd = ["tar", "-x", "-O", "-f", "-", "." + conf_file]
md5_cmd = ["md5sum"]
dpkg_p = Popen(dpkg_cmd, stdout=PIPE)
tar_p = Popen(tar_cmd, stdin=dpkg_p.stdout, stdout=PIPE,
universal_newlines=True)
md5_p = Popen(md5_cmd, stdin=tar_p.stdout, stdout=PIPE,
universal_newlines=True)
pkg_md5sum = md5_p.communicate()[0].split()[0]
for __p in [dpkg_p, tar_p, md5_p]:
p = cast(Popen, __p)
p.stdout.close()
p.wait()
return pkg_md5sum
def get_md5sum_for_file_installed(conf_file, prefix):
# type: (str, str) -> str
try:
with open(prefix + conf_file, 'rb') as fb:
for hash_string in apt_pkg.Hashes(fb).hashes: # type: ignore
if hash_string.hashtype == 'MD5Sum':
return hash_string.hashvalue
return None
except IsADirectoryError:
# the package replaces a directory wih a configuration file
#
# if the package changed this way it is safe to assume that
# the transition happens without showing a prompt but if the admin
# created the directory the admin will need to resolve it after
# being notified about the unexpected prompt
logging.debug("found conffile %s is a directory on the system "
% conf_file)
return "dir"
except FileNotFoundError:
# if the local file got deleted by the admin thats ok but it may still
# trigger a conffile promp (see debian #788049)
logging.debug("conffile %s in missing on the system" % conf_file)
return ""
def map_conf_file(conf_file, conffiles):
# type: (str, Union[AbstractSet[str], Dict[str, str]]) -> str
"""Find respective conffile in a set of conffiles with some heuristics
"""
if conf_file in conffiles:
return conf_file
elif os.path.join(conf_file, os.path.basename(conf_file)) in conffiles:
# new /etc/foo may be old /etc/foo/foo, like in LP: #1822745
return os.path.join(conf_file, os.path.basename(conf_file))
elif os.path.dirname(conf_file) in conffiles:
# new /etc/foo/foo may be old /etc/foo, probably by accident
return os.path.dirname(conf_file)
# TODO: peek into package's dpkg-maintscript-helper mv_conffile usage
else:
return None
# prefix is *only* needed for the build-in tests
def conffile_prompt(destFile, prefix=""):
# type: (str, str) -> bool
logging.debug("check_conffile_prompt(%s)" % destFile)
pkgname = pkgname_from_deb(destFile)
# get the conffiles for the /var/lib/dpkg/status file
status_file = apt_pkg.config.find("Dir::State::status")
with open(status_file, "r") as f:
tagfile = apt_pkg.TagFile(f)
conffiles = ""
for section in tagfile:
if section.get("Package") == pkgname:
logging.debug("found pkg: %s" % pkgname)
if "Conffiles" in section:
conffiles = section.get("Conffiles")
break
# get conffile value from pkg, its ok if the new version
# does not have conffiles anymore
pkg_conffiles = set() # type: AbstractSet[str]
try:
deb = apt_inst.DebFile(destFile)
pkg_conffiles = set(deb.control.extractdata(
"conffiles").strip().decode("utf-8").split("\n"))
except SystemError as e:
print(_("Apt returned an error, exiting"))
print(_("error message: %s") % e)
logging.error(_("Apt returned an error, exiting"))
logging.error(_("error message: %s"), e)
raise
except LookupError as e:
logging.debug("No conffiles in deb %s (%s)" % (destFile, e))
if not pkg_conffiles:
return False
# Conffiles:
# /etc/bash_completion.d/m-a c7780fab6b14d75ca54e11e992a6c11c
dpkg_status_conffiles = {}
for line in conffiles.splitlines():
# ignore empty lines
line = line.strip()
if not line:
continue
# show what we do
logging.debug("conffile line: %s", line)
li = line.split()
conf_file = li[0]
md5 = li[1]
if len(li) > 2:
obs = li[2]
else:
obs = None
# ignore if conffile is obsolete
if obs == "obsolete":
continue
# ignore state "newconffile" until its clearer if there
# might be a dpkg prompt (LP: #936870)
if md5 == "newconffile":
continue
new_conf_file = map_conf_file(conf_file, pkg_conffiles)
if not new_conf_file:
logging.debug("%s not in package conffiles %s" % (
conf_file, pkg_conffiles))
continue
# record for later
dpkg_status_conffiles[conf_file] = md5
# test against the installed file, if the local file got deleted
# by the admin thats ok but it may still trigger a conffile prompt
# (see debian #788049)
current_md5 = get_md5sum_for_file_installed(conf_file, prefix)
logging.debug("current md5: %s" % current_md5)
# hashes are the same, no conffile prompt
if current_md5 == md5:
continue
# calculate md5sum from the deb (may take a bit)
pkg_md5sum = get_md5sum_for_file_in_deb(destFile, new_conf_file)
logging.debug("pkg_md5sum: %s" % pkg_md5sum)
# the md5sum in the deb is unchanged, this will not
# trigger a conffile prompt
if pkg_md5sum == md5:
continue
# if we made it to this point:
# current_md5 != pkg_md5sum != md5
# and that will trigger a conffile prompt, we can
# stop processing at this point and just return True
return True
# now check if there are conffiles in the pkg that where not there
# in the previous version in the dpkg status file
if pkg_conffiles:
for conf_file in pkg_conffiles:
old_conf_file = map_conf_file(conf_file, dpkg_status_conffiles)
if not old_conf_file:
pkg_md5sum = get_md5sum_for_file_in_deb(destFile, conf_file)
current_md5 = get_md5sum_for_file_installed(conf_file, prefix)
if current_md5 != "" and pkg_md5sum != current_md5:
return True
return False
def dpkg_conffile_prompt():
# type: () -> bool
if "DPkg::Options" not in apt_pkg.config:
return True
options = apt_pkg.config.value_list("DPkg::Options")
for option in options:
option = option.strip()
if option in ["--force-confold", "--force-confnew"]:
return False
return True
def rewind_cache(cache, pkgs_to_upgrade):
# type: (UnattendedUpgradesCache, List[apt.Package]) -> None
""" set the cache back to the state with packages_to_upgrade """
cache.clear()
for pkg2 in pkgs_to_upgrade:
cache.mark_install_adjusted(pkg2, from_user=not pkg2.is_auto_installed)
if cache.broken_count > 0:
raise AssertionError("rewind_cache created a broken cache")
def host():
# type: () -> str
return socket.getfqdn()
def wrap_indent(t, subsequent_indent=" "):
# type: (str, str) -> str
return "\n".join(wrap(t, break_on_hyphens=False,
subsequent_indent=subsequent_indent))
def setup_apt_listchanges(conf="/etc/apt/listchanges.conf"):
# type: (str) -> None
""" deal with apt-listchanges """
# apt-listchanges will always send a mail if there is a mail address
# set in the config regardless of the frontend used, so set it to
# mail if we have a sendmail and to none if not (as it appears to
# not check if sendmail is there or not), debian bug #579733
if os.path.exists(SENDMAIL_BINARY):
os.environ["APT_LISTCHANGES_FRONTEND"] = "mail"
else:
os.environ["APT_LISTCHANGES_FRONTEND"] = "none"
def _send_mail_using_mailx(from_address, to_address, subject, body):
# type: (str, str, str, str) -> int
# ensure that the body is a byte stream and that we do not
# break on encoding errors (the default error mode is "strict")
encoded_body = body.encode(
locale.getpreferredencoding(False), errors="replace")
# we use a binary pipe to stdin to ensure we do not break on
# unicode encoding errors (e.g. because the user is running a
# ascii only system like the buildds)
mail = subprocess.Popen(
[MAIL_BINARY, "-r", from_address, "-s", subject, to_address],
stdin=subprocess.PIPE, universal_newlines=False)
mail.stdin.write(encoded_body)
mail.stdin.close()
ret = mail.wait()
return ret
def _send_mail_using_sendmail(from_address, to_address, subject, body):
# type: (str, str, str, str) -> int
# format as a proper mail
msg = Message()
msg['Subject'] = subject
msg['From'] = from_address
msg['To'] = to_address
msg['Auto-Submitted'] = "auto-generated"
# order is important here, Message() first, then Charset()
# then msg.set_charset()
charset = email.charset.Charset("utf-8")
charset.body_encoding = email.charset.QP # type: ignore
msg.set_payload(body, charset)
# and send it away
sendmail = subprocess.Popen(
[SENDMAIL_BINARY, "-oi", "-t"],
stdin=subprocess.PIPE, universal_newlines=True)
sendmail.stdin.write(msg.as_string())
sendmail.stdin.close()
ret = sendmail.wait()
return ret
def send_summary_mail(pkgs, # type: List[str]
res, # type: bool
result_str, # type: str
pkgs_kept_back, # type: KeptPkgs
pkgs_removed, # type: List[str]
pkgs_kept_installed, # type: List[str]
mem_log, # type: StringIO
dpkg_log_content, # type: str
):
# type: (...) -> None
""" send mail (if configured in Unattended-Upgrade::Mail) """
to_email = apt_pkg.config.find("Unattended-Upgrade::Mail", "")
if not to_email:
return
if not os.path.exists(MAIL_BINARY) and not os.path.exists(SENDMAIL_BINARY):
logging.error(_("No /usr/bin/mail or /usr/sbin/sendmail, "
"can not send mail. "
"You probably want to install the mailx package."))
return
# The admin may well wish to get a mail report regardless of what was done.
# This is now set by Unattended-Upgrade::MailReport values of:
# "always", "only-on-error" or "on-change"
# (you can achieve "never" by not setting Unattended-Upgrade::Mail).
# If this is not set, then set it using any legacy MailOnlyOnError
# setting (default True)
#
mail_opt = apt_pkg.config.find("Unattended-Upgrade::MailReport")
if (mail_opt == ""): # None set - map from legacy value
if apt_pkg.config.find_b("Unattended-Upgrade::MailOnlyOnError", False):
mail_opt = "only-on-error"
else:
mail_opt = "on-change"
# if the operation was successful and the user has requested to get
# mails only on errors, just exit here
if (res and (mail_opt == "only-on-error")):
return
# if the run was successful but nothing had to be done skip sending email
# unless the admin wants it anyway
if (((mail_opt != "always") and res and not pkgs and not pkgs_kept_back
and not pkgs_removed)):
return
# Check if reboot-required flag is present
reboot_flag_str = _(
"[reboot required]") if os.path.isfile(REBOOT_REQUIRED_FILE) else ""
# Check if packages are kept on hold
hold_flag_str = (_("[package on hold]") if pkgs_kept_back
or pkgs_kept_installed else "")
logging.debug("Sending mail to %s" % to_email)
subject = _(
"{hold_flag}{reboot_flag} unattended-upgrades result for "
"{machine}: {result}").format(
hold_flag=hold_flag_str, reboot_flag=reboot_flag_str,
machine=host(), result="SUCCESS" if res else "FAILURE").strip()
body = wrap_indent(_("Unattended upgrade result: %s") % result_str)
body += "\n\n"
if os.path.isfile(REBOOT_REQUIRED_FILE):
body += _(
"Warning: A reboot is required to complete this upgrade, "
"or a previous one.\n\n")
if pkgs:
if res:
body += _("Packages that were upgraded:\n")
else:
body += _("Packages that attempted to upgrade:\n")
body += " " + wrap_indent(" ".join(pkgs))
body += "\n\n"
if pkgs_kept_back:
body += _("Packages with upgradable origin but kept back:\n")
for origin, origin_pkgs in pkgs_kept_back.items():
body += " " + origin + ":\n"
body += " " + wrap_indent(" ".join(origin_pkgs),
subsequent_indent=" ") + "\n"
body += "\n"
if pkgs_removed:
body += _("Packages that were auto-removed:\n")
body += " " + wrap_indent(" ".join(pkgs_removed))
body += "\n\n"
if pkgs_kept_installed:
body += _("Packages that were kept from being auto-removed:\n")
body += " " + wrap_indent(" ".join(pkgs_kept_installed))
body += "\n\n"
if dpkg_log_content:
body += _("Package installation log:") + "\n"
body += dpkg_log_content
body += "\n\n"
body += _("Unattended-upgrades log:\n")
body += mem_log.getvalue()
from_email = apt_pkg.config.find("Unattended-Upgrade::Sender", "root")
if os.path.exists(SENDMAIL_BINARY):
ret = _send_mail_using_sendmail(from_email, to_email, subject, body)
elif os.path.exists(MAIL_BINARY):
ret = _send_mail_using_mailx(from_email, to_email, subject, body)
else:
raise AssertionError(
"This should never be reached as we previously validated that we "
"either have sendmail or mailx. Maybe they've been removed in "
"this right moment?")
logging.debug("mail returned: %s", ret)
def do_install(cache, # type: UnattendedUpgradesCache
pkgs_to_upgrade, # type: List[str]
options, # type: Options
logfile_dpkg, # type: str
):
# type: (...) -> bool
setup_apt_listchanges()
logging.info(_("Writing dpkg log to %s"), logfile_dpkg)
if cache.get_changes():
cache.clear()
pkg_install_success = False
try:
if options.minimal_upgrade_steps:
# try upgrade all "pkgs" in minimal steps
pkg_install_success = upgrade_in_minimal_steps(
cache, pkgs_to_upgrade,
logfile_dpkg,
options.verbose or options.debug)
else:
mark_pkgs_to_upgrade(cache, pkgs_to_upgrade)
pkg_install_success = upgrade_normal(
cache, logfile_dpkg, options.verbose or options.debug)
except Exception as e:
# print unhandled exceptions here this way, while stderr is redirected
os.write(2, ("Exception: %s\n" % e).encode('utf-8'))
pkg_install_success = False
return pkg_install_success
def _setup_alternative_rootdir(rootdir):
# type: (str) -> None
# clear system unattended-upgrade stuff
apt_pkg.config.clear("Unattended-Upgrade")
# read rootdir (taken from apt.Cache, but we need to run it
# here before the cache gets initialized
if os.path.exists(rootdir + "/etc/apt/apt.conf"):
apt_pkg.read_config_file(apt_pkg.config,
rootdir + "/etc/apt/apt.conf")
if os.path.isdir(rootdir + "/etc/apt/apt.conf.d"):
apt_pkg.read_config_dir(apt_pkg.config,
rootdir + "/etc/apt/apt.conf.d")
logdir = os.path.join(rootdir, "var", "log", "unattended-upgrades")
if not os.path.exists(logdir):
os.makedirs(logdir)
apt.apt_pkg.config.set("Unattended-Upgrade::LogDir", logdir)
def _get_logdir():
# type: () -> str
logdir = apt_pkg.config.find_dir(
"Unattended-Upgrade::LogDir",
# COMPAT only
apt_pkg.config.find_dir("APT::UnattendedUpgrades::LogDir",
"/var/log/unattended-upgrades/"))
return logdir
def _setup_logging(options):
# type: (Options) -> StringIO
# ensure this is run only once
if len(logging.root.handlers) > 0:
return None
# init the logging
logdir = _get_logdir()
logfile = os.path.join(
logdir,
apt_pkg.config.find(
"Unattended-Upgrade::LogFile",
# COMPAT only
apt_pkg.config.find("APT::UnattendedUpgrades::LogFile",
"unattended-upgrades.log")))
if not options.dry_run and not os.path.exists(logdir):
os.makedirs(logdir)
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s',
filename=logfile)
# additional logging
logger = logging.getLogger()
mem_log = StringIO()
if options.apt_debug:
apt_pkg.config.set("Debug::pkgProblemResolver", "1")
apt_pkg.config.set("Debug::pkgDepCache::AutoInstall", "1")
if options.debug:
logger.setLevel(logging.DEBUG)
stdout_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stdout_handler)
elif options.verbose:
logger.setLevel(logging.INFO)
stdout_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stdout_handler)
if apt_pkg.config.find("Unattended-Upgrade::Mail", ""):
mem_log_handler = logging.StreamHandler(mem_log)
logger.addHandler(mem_log_handler)
# Configure syslog if necessary
syslogEnable = apt_pkg.config.find_b("Unattended-Upgrade::SyslogEnable",
False)
if syslogEnable:
syslogFacility = apt_pkg.config.find(
"Unattended-Upgrade::SyslogFacility",
"daemon")
syslogHandler = logging.handlers.SysLogHandler(
address='/dev/log',
facility=syslogFacility) # type: ignore
syslogHandler.setFormatter(
logging.Formatter("unattended-upgrade: %(message)s"))
known = syslogHandler.facility_names.keys() # type: ignore
if syslogFacility.lower() in known:
logger.addHandler(syslogHandler)
logging.info("Enabled logging to syslog via %s facility "
% syslogFacility)
else:
logging.warning("Syslog facility %s was not found"
% syslogFacility)
return mem_log
def logged_in_users():
# type: () -> AbstractSet[str]
"""Return a list of logged in users"""
# the "users" command always returns a single line with:
# "user1, user1, user2"
users = subprocess.check_output(
USERS, universal_newlines=True).rstrip('\n')
return set(users.split())
def reboot_if_requested_and_needed():
# type: () -> None
"""auto-reboot (if required and the config for this is set)"""
if not os.path.exists(REBOOT_REQUIRED_FILE):
return
if not apt_pkg.config.find_b(
"Unattended-Upgrade::Automatic-Reboot", False):
return
# see if we need to check for logged in users
if not apt_pkg.config.find_b(
"Unattended-Upgrade::Automatic-Reboot-WithUsers", True):
users = logged_in_users()
if users:
msg = gettext.ngettext(
"Found %s, but not rebooting because %s is logged in." % (
REBOOT_REQUIRED_FILE, users),
"Found %s, but not rebooting because %s are logged in." % (
REBOOT_REQUIRED_FILE, users),
len(users))
logging.warning(msg)
return
# reboot at the specified time
when = apt_pkg.config.find(
"Unattended-Upgrade::Automatic-Reboot-Time", "now")
logging.warning("Found %s, rebooting" % REBOOT_REQUIRED_FILE)
cmd = ["/sbin/shutdown", "-r", when]
try:
shutdown_msg = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
if shutdown_msg.strip():
logging.warning("Shutdown msg: %s", shutdown_msg.strip())
except Exception as e:
logging.error("Failed to issue shutdown: %s", e)
def write_stamp_file():
# type: () -> None
statedir = os.path.join(apt_pkg.config.find_dir("Dir::State"), "periodic")
if not os.path.exists(statedir):
os.makedirs(statedir)
with open(os.path.join(statedir, "unattended-upgrades-stamp"), "w"):
pass
def try_to_upgrade(pkg, # type: apt.Package
pkgs_to_upgrade, # type: List[apt.Package]
cache, # type: UnattendedUpgradesCache
):
# type: (...) -> None
try:
try:
# try to adjust pkg itself first, if that throws an exception it
# can't be upgraded on its own
cache.adjust_candidate(pkg)
if not pkg.is_upgradable and not apt_pkg.config.find_b(
"Unattended-Upgrade::Allow-downgrade", False):
return
except NoAllowedOriginError:
return
cache._cached_candidate_pkgnames.add(pkg.name)
cache.mark_upgrade_adjusted(pkg, from_user=not pkg.is_auto_installed)
if check_changes_for_sanity(cache, pkg):
# add to packages to upgrade
pkgs_to_upgrade.append(pkg)
else:
rewind_cache(cache, pkgs_to_upgrade)
except (SystemError, NoAllowedOriginError) as e:
# can't upgrade
logging.warning(
_("package %s upgradable but fails to "
"be marked for upgrade (%s)"), pkg.name, e)
rewind_cache(cache, pkgs_to_upgrade)
def candidate_version_changed(pkg, # type: apt.Package
):
return (pkg.is_installed and pkg.candidate
and pkg.candidate.version != pkg.installed.version
and apt_pkg.config.find_b(
'Unattended-Upgrade::Allow-downgrade', False))
def calculate_upgradable_pkgs(cache, # type: UnattendedUpgradesCache
options, # type: Options
):
# type: (...) -> List[apt.Package]
pkgs_to_upgrade = [] # type: List[apt.Package]
# now do the actual upgrade
for pkg in cache:
if options.debug and pkg.is_upgradable \
or candidate_version_changed(pkg):
logging.debug("Checking: %s (%s)" % (
pkg.name, getattr(pkg.candidate, "origins", [])))
if (pkg.is_upgradable or candidate_version_changed(pkg)
and is_pkgname_in_whitelist(pkg.name, cache.whitelist)):
try:
ver_in_allowed_origin(pkg, cache.allowed_origins)
except NoAllowedOriginError:
continue
try_to_upgrade(pkg,
pkgs_to_upgrade,
cache)
if cache.get_changes():
cache.clear()
return pkgs_to_upgrade
def get_dpkg_log_content(logfile_dpkg, install_start_time):
# type: (str, datetime.datetime) -> str
logging.debug("Extracting content from %s since %s" % (
logfile_dpkg, install_start_time))
content = []
found_start = False
try:
with io.open(logfile_dpkg, encoding='utf-8', errors='replace') as fp:
# read until we find the last "Log started: "
for line in fp.readlines():
# scan for the first entry we need (minimal-step mode
# creates a new stanza for each individual install)
if not found_start and line.startswith("Log started: "):
stanza_start = LoggingDateTime.from_string(
line[len("Log started: "):-1])
if stanza_start >= install_start_time:
found_start = True
if found_start:
# skip progress indicator until #860931 is fixed in apt
# and dpkg
if re.match(
"^\\(Reading database \\.\\.\\. ()|([0-9]+%)$",
line):
continue
content.append(line)
return "".join(content)
except FileNotFoundError:
return ""
def get_auto_removable(cache):
# type: (apt.Cache) -> AbstractSet[str]
return {pkg.name for pkg in cache
if pkg.is_auto_removable}
def is_autoremove_valid(cache, # type: UnattendedUpgradesCache
pkgname, # type: str
auto_removable, # type: AbstractSet[str]
):
# type: (...) -> bool
changes = cache.get_changes()
if not changes:
# package is already removed
return True
pkgnames = {pkg.name for pkg in changes}
for pkg in changes:
if not is_pkg_change_allowed(pkg, cache.blacklist, cache.whitelist,
cache.strict_whitelist):
logging.warning(
_("Keeping the following auto-removable package(s) because "
"they include %s which is set to be kept unmodified: %s"),
pkg.name, " ".join(sorted(pkgnames)))
return False
if not pkgnames.issubset(auto_removable):
if pkgname != "":
logging.warning(
_("Keeping auto-removable %s package(s) because it would"
" also remove the following packages which should "
"be kept in this step: %s"), pkgname,
" ".join(sorted(pkgnames - auto_removable)))
else:
logging.warning(
_("Keeping %s auto-removable package(s) because it would"
" also remove the following packages which should "
"be kept in this step: %s"), len(auto_removable),
" ".join(sorted(pkgnames - auto_removable)))
return False
for packagename in pkgnames:
if cache.running_kernel_pkgs_regexp and \
cache.running_kernel_pkgs_regexp.match(packagename):
logging.warning(
_("Keeping the following auto-removable package(s) because "
"they include %s which package is related to the running "
"kernel: %s"), packagename, " ".join(sorted(pkgnames)))
return False
if cache.install_count > 0:
logging.error(
"The following packages are marked for installation or upgrade "
"which is not allowed when performing autoremovals: %s",
" ".join([pkg.name for pkg in changes if not pkg.marked_delete]))
return False
return True
def do_auto_remove(cache, # type: UnattendedUpgradesCache
auto_removable, # type: AbstractSet[str]
logfile_dpkg, # type: str
minimal_steps, # type: bool
verbose=False, # type: bool
dry_run=False # type: bool
):
# type: (...) -> Tuple[bool, List[str], List[str]]
res = True
if not auto_removable:
return (res, [], [])
pkgs_removed = [] # type: List[str]
pkgs_kept_installed = [] # type: List[str]
if minimal_steps:
for pkgname in auto_removable:
if should_stop():
pkgs_kept_installed = list(auto_removable - set(pkgs_removed))
return (False, pkgs_removed, pkgs_kept_installed)
logging.debug("marking %s for removal" % pkgname)
if pkgname in pkgs_removed:
continue
cache[pkgname].mark_delete()
if not is_autoremove_valid(cache, pkgname, auto_removable):
# this situation can occur when removing newly unused packages
# would also remove old unused packages which are not set
# for removal, thus getting there is not handled as an error
pkgs_kept_installed.append(pkgname)
cache.clear()
continue
if not dry_run:
changes = cache.get_changes()
pkgnames = {pkg.name for pkg in changes}
res, error = cache_commit(cache, logfile_dpkg, verbose)
if not res:
break
pkgs_removed.extend(pkgnames)
else:
cache.clear()
else:
for pkgname in auto_removable:
cache[pkgname].mark_delete()
if is_autoremove_valid(cache, "", auto_removable):
# do it in one step
if not dry_run:
res, error = cache_commit(cache, logfile_dpkg, verbose)
else:
cache.clear()
else:
cache.clear()
if res:
logging.info(_("Packages that were successfully auto-removed: %s"),
" ".join(sorted(pkgs_removed)))
logging.info(_("Packages that are kept back: %s"),
" ".join(sorted(pkgs_kept_installed)))
if not res:
cache.clear()
logging.error(_("Auto-removing the packages failed!"))
logging.error(_("Error message: %s"), error)
logging.error(_("dpkg returned an error! See %s for details"),
logfile_dpkg)
return (res, pkgs_removed, pkgs_kept_installed)
def clean_downloaded_packages(fetcher):
# type: (apt_pkg.Acquire) -> None
archivedir = os.path.dirname(
apt_pkg.config.find_dir("Dir::Cache::archives"))
for item in fetcher.items:
if os.path.dirname(os.path.abspath(item.destfile)) == archivedir:
try:
os.unlink(item.destfile)
except OSError:
pass
def is_update_day():
# type: () -> bool
# check if patch days are configured
patch_days = apt_pkg.config.value_list("Unattended-Upgrade::Update-Days")
if not patch_days:
return True
# validate patch days
today = date.today()
# abbreviated localized dayname
if today.strftime("%a") in patch_days:
return True
# full localized dayname
if today.strftime("%A") in patch_days:
return True
# by number (Sun: 0, Mon: 1, ...)
if today.strftime("%w") in patch_days:
return True
# today is not a patch day
logging.info(
"Skipping update check: today is %s,%s,%s but patch days are %s",
today.strftime("%w"), today.strftime("%a"), today.strftime("%A"),
", ".join(patch_days))
return False
def update_kept_pkgs_file(kept_pkgs, kept_file):
# type: (DefaultDict[str, List[str]], str) -> None
if kept_pkgs:
pkgs_all_origins = set()
for origin_pkgs in kept_pkgs.values():
pkgs_all_origins.update(origin_pkgs)
try:
with open(kept_file, "w") as kf:
kf.write(" ".join(sorted(pkgs_all_origins)))
except FileNotFoundError:
logging.error(_("Could not open %s for saving list of packages "
"kept back." % kept_file))
else:
if os.path.exists(kept_file):
os.remove(kept_file)
def main(options, rootdir="/"):
# type: (Options, str) -> int
# useful for testing
if not rootdir == "/":
_setup_alternative_rootdir(rootdir)
# see debian #776752
install_start_time = datetime.datetime.now().replace(microsecond=0)
# setup logging
mem_log = _setup_logging(options)
# get log
logfile_dpkg = os.path.join(_get_logdir(), 'unattended-upgrades-dpkg.log')
if not os.path.exists(logfile_dpkg):
with open(logfile_dpkg, 'w'):
pass
# lock for the shutdown check
shutdown_lock = apt_pkg.get_lock(LOCK_FILE)
if shutdown_lock < 0:
logging.error("Lock file is already taken, exiting")
return 1
try:
res = run(options, rootdir, mem_log, logfile_dpkg,
install_start_time)
if res.success and res.result_str:
# complete, successful run
update_kept_pkgs_file(res.pkgs_kept_back,
os.path.join(rootdir, KEPT_PACKAGES_FILE))
if res.result_str and not options.dry_run:
# there is some meaningful result which is worth an email
log_content = get_dpkg_log_content(logfile_dpkg,
install_start_time)
send_summary_mail(res.pkgs, res.success, res.result_str,
res.pkgs_kept_back, res.pkgs_removed,
res.pkgs_kept_installed, mem_log,
log_content)
if res.update_stamp:
# write timestamp file
write_stamp_file()
if not options.dry_run:
# check if the user wants a reboot
reboot_if_requested_and_needed()
os.close(shutdown_lock)
if res.success:
return 0
else:
return 1
except Exception as e:
logger = logging.getLogger()
logger.exception(_("An error occurred: %s"), e)
log_content = get_dpkg_log_content(logfile_dpkg,
install_start_time)
if not options.dry_run:
send_summary_mail(["<unknown>"], False, _("An error occurred"),
None, [], [], mem_log, log_content)
# Re-raise exceptions for apport
raise
def mark_pkgs_to_upgrade(cache, pkgs_to_upgrade):
# type (apt.Cache, List[str]) -> None
for pkg_name in pkgs_to_upgrade:
pkg = cache[pkg_name]
if pkg.is_upgradable \
or (pkg.is_installed
and pkg.candidate.version != pkg.installed.version) \
and apt_pkg.config.find_b("Unattended-Upgrade::Allow-downgrade",
False):
cache.mark_upgrade_adjusted(pkg,
from_user=not pkg.is_auto_installed)
elif not pkg.is_installed:
cache.mark_install_adjusted(pkg, from_user=False)
def run(options, # type: Options
rootdir, # type: str
mem_log, # type: StringIO
logfile_dpkg, # type: str
install_start_time, # type: datetime.datetime
):
# type: (...) -> UnattendedUpgradesResult
# check if today is a patch day
if not is_update_day():
return UnattendedUpgradesResult(True)
# check if u-u should be stopped already
if should_stop():
return UnattendedUpgradesResult(False)
# check to see if want to auto-upgrade the devel release
if apt_pkg.config.find("Unattended-Upgrade::DevRelease") == "auto":
try:
if DISTRO_ID.lower() == 'ubuntu':
devel = (distro_info.UbuntuDistroInfo() .
devel(result="object"))
elif DISTRO_ID.lower() == 'debian':
devel = (distro_info.DebianDistroInfo() .
devel(result="object"))
else:
devel = (distro_info.DistroInfo(DISTRO_ID) .
devel(result="object"))
except Exception as e:
logging.warning("Could not figure out development release: %s" % e)
else:
if ((devel.series == DISTRO_CODENAME
and devel.release is not None
and devel.release - date.today() > DEVEL_UNTIL_RELEASE)):
syslog.syslog((_("Not running on this development "
"release before %s") %
(devel.release - DEVEL_UNTIL_RELEASE
- datetime.timedelta(days=1))))
logging.warning(_("Not running on this development "
"release before %s") %
(devel.release - DEVEL_UNTIL_RELEASE
- datetime.timedelta(days=1)))
return UnattendedUpgradesResult(True)
logging.debug("Running on the development release")
elif "(development branch)" in DISTRO_DESC and not\
apt_pkg.config.find_b("Unattended-Upgrade::DevRelease", True):
syslog.syslog(_("Not running on the development release."))
logging.info(_("Not running on the development release."))
return UnattendedUpgradesResult(True)
logging.info(_("Starting unattended upgrades script"))
# check and get lock
try:
apt_pkg.pkgsystem_lock()
except SystemError:
logging.error(_("Lock could not be acquired (another package "
"manager running?)"))
print(_("Cache lock can not be acquired, exiting"))
return UnattendedUpgradesResult(
False, _("Lock could not be acquired"))
# check if the journal is dirty and if so, take emergceny action
# the alternative is to leave the system potentially unsecure until
# the user comes in and fixes
if is_dpkg_journal_dirty() and \
apt_pkg.config.find_b("Unattended-Upgrade::AutoFixInterruptedDpkg",
True):
logging.warning(
_("Unclean dpkg state detected, trying to correct"))
print(_("Unclean dpkg state detected, trying to correct"))
env = copy.copy(os.environ)
env["DPKG_FRONTEND_LOCKED"] = "1"
try:
with Unlocked():
output = subprocess.check_output(
["dpkg", "--force-confold", "--configure", "-a"],
env=env,
universal_newlines=True)
except subprocess.CalledProcessError as e:
output = e.output
logging.warning(_("dpkg --configure -a output:\n%s"), output)
# get a cache
try:
cache = UnattendedUpgradesCache(rootdir=rootdir)
except SystemError as error:
print(_("Apt returned an error, exiting"))
print(_("error message: %s") % error)
logging.error(_("Apt returned an error, exiting"))
logging.error(_("error message: %s"), error)
return UnattendedUpgradesResult(
False, _("Apt returned an error, exiting"))
if cache._depcache.broken_count > 0:
print(_("Cache has broken packages, exiting"))
logging.error(_("Cache has broken packages, exiting"))
return UnattendedUpgradesResult(
False, _("Cache has broken packages, exiting"))
# FIXME: make this into a ContextManager
# be nice when calculating the upgrade as its pretty CPU intensive
old_priority = os.nice(0)
try:
# Check that we will be able to restore the priority
os.nice(-1)
os.nice(20)
except OSError as e:
if e.errno in (errno.EPERM, errno.EACCES):
pass
else:
raise
auto_removable = get_auto_removable(cache)
# find out about the packages that are upgradable (in an allowed_origin)
pkgs_to_upgrade = calculate_upgradable_pkgs(cache, options)
pkgs_to_upgrade.sort(key=lambda p: p.name)
pkgs = [pkg.name for pkg in pkgs_to_upgrade]
logging.debug("pkgs that look like they should be upgraded: %s"
% "\n".join(pkgs))
# FIXME: make this into a ContextManager
# stop being nice
os.nice(old_priority - os.nice(0))
# download what looks good
mark_pkgs_to_upgrade(cache, pkgs)
if options.debug:
fetcher = apt_pkg.Acquire(apt.progress.text.AcquireProgress())
else:
fetcher = apt_pkg.Acquire()
list = apt_pkg.SourceList()
list.read_main_list()
recs = cache._records
pm = apt_pkg.PackageManager(cache._depcache)
# don't start downloading during shutdown
# TODO: download files one by one and check for stop request after each of
# them
if should_stop():
return UnattendedUpgradesResult(False, _("Upgrade was interrupted"))
try:
pm.get_archives(fetcher, list, recs)
except SystemError as e:
logging.error(_("GetArchives() failed: %s"), e)
try:
res = fetcher.run()
logging.debug("fetch.run() result: %s", res)
except SystemError as e:
logging.error("fetch.run() result: %s", e)
if options.download_only:
return UnattendedUpgradesResult(True)
if cache.get_changes():
cache.clear()
pkg_conffile_prompt = False
if dpkg_conffile_prompt():
# now check the downloaded debs for conffile conflicts and build
# a blacklist
conffile_blacklist = [] # type: List[str]
for item in fetcher.items:
logging.debug("%s" % item)
if item.status == item.STAT_ERROR:
print(_("An error occurred: %s") % item.error_text)
logging.error(_("An error occurred: %s"), item.error_text)
if not item.complete:
print(_("The URI %s failed to download, aborting") %
item.desc_uri)
logging.error(_("The URI %s failed to download, aborting"),
item.desc_uri)
return UnattendedUpgradesResult(
False, (_("The URI %s failed to download, aborting") %
item.desc_uri))
if not os.path.exists(item.destfile):
print(_("Download finished, but file %s not there?!?") %
item.destfile)
logging.error("Download finished, but file %s not "
"there?!?", item.destfile)
return UnattendedUpgradesResult(
False, (_("Download finished, but file %s not there?!?") %
item.destfile))
if not item.is_trusted and not apt_pkg.config.find_b(
"APT::Get::AllowUnauthenticated", False):
logging.debug("%s is blacklisted because it is not trusted")
pkg_name = pkgname_from_deb(item.destfile)
if not is_pkgname_in_blacklist(pkg_name, cache.blacklist):
conffile_blacklist.append("%s$" % re.escape(pkg_name))
if not is_deb(item.destfile):
logging.debug("%s is not a .deb file" % item)
continue
if conffile_prompt(item.destfile):
# skip package (means to re-run the whole marking again
# and making sure that the package will not be pulled in by
# some other package again!)
#
# print to stdout to ensure that this message is part of
# the cron mail (only if no summary mail is requested)
email = apt_pkg.config.find("Unattended-Upgrade::Mail", "")
if not email:
print(_("Package %s has conffile prompt and needs "
"to be upgraded manually") %
pkgname_from_deb(item.destfile))
# log to the logfile
logging.warning(_("Package %s has conffile prompt and "
"needs to be upgraded manually"),
pkgname_from_deb(item.destfile))
pkg_name = pkgname_from_deb(item.destfile)
if not is_pkgname_in_blacklist(pkg_name, cache.blacklist):
conffile_blacklist.append("%s$" % re.escape(pkg_name))
pkg_conffile_prompt = True
# redo the selection about the packages to upgrade based on the new
# blacklist
logging.debug("Packages blacklist due to conffile prompts: %s"
% conffile_blacklist)
# find out about the packages that are upgradable (in a allowed_origin)
if len(conffile_blacklist) > 0:
for regex in conffile_blacklist:
cache.blacklist.append(regex)
cache.apply_pinning(cache.pinning_from_regex_list(
conffile_blacklist, NEVER_PIN)) # type: ignore
old_pkgs_to_upgrade = pkgs_to_upgrade[:]
pkgs_to_upgrade = []
for pkg in old_pkgs_to_upgrade:
logging.debug("Checking the black and whitelist: %s" %
(pkg.name))
cache.mark_upgrade_adjusted(
pkg, from_user=not pkg.is_auto_installed)
if check_changes_for_sanity(cache):
pkgs_to_upgrade.append(pkg)
else:
logging.info(_("package %s not upgraded"), pkg.name)
cache.clear()
for pkg2 in pkgs_to_upgrade:
cache.call_adjusted(
apt.package.Package.mark_upgrade, pkg2,
from_user=not pkg2.is_auto_installed)
if cache.get_changes():
cache.clear()
else:
logging.debug("dpkg is configured not to cause conffile prompts")
# auto-removals
kernel_pkgs_remove_success = True # type: bool
kernel_pkgs_removed = [] # type: List[str]
kernel_pkgs_kept_installed = [] # type: List[str]
if (auto_removable and apt_pkg.config.find_b(
"Unattended-Upgrade::Remove-Unused-Kernel-Packages", True)):
# remove unused kernels before installing new ones because the newly
# installed ones may fill up /boot and break the system right before
# removing old ones could take place
#
# this step may also remove _auto-removable_ reverse dependencies
# of kernel packages
auto_removable_kernel_pkgs = {
p for p in auto_removable
if (cache.versioned_kernel_pkgs_regexp
and cache.versioned_kernel_pkgs_regexp.match(p)
and not cache.running_kernel_pkgs_regexp.match(p))}
if auto_removable_kernel_pkgs:
logging.info(_("Removing unused kernel packages: %s"),
" ".join(auto_removable_kernel_pkgs))
(kernel_pkgs_remove_success,
kernel_pkgs_removed,
kernel_pkgs_kept_installed) = do_auto_remove(
cache, auto_removable_kernel_pkgs, logfile_dpkg,
options.minimal_upgrade_steps,
options.verbose or options.debug, options.dry_run)
auto_removable = get_auto_removable(cache)
previous_autoremovals = auto_removable
if apt_pkg.config.find_b(
"Unattended-Upgrade::Remove-Unused-Dependencies", False):
pending_autoremovals = previous_autoremovals
else:
pending_autoremovals = set()
# exit if there is nothing to do and nothing to report
if (len(pending_autoremovals) == 0
and len(pkgs_to_upgrade) == 0):
logging.info(_("No packages found that can be upgraded unattended "
"and no pending auto-removals"))
pkgs_kept_back = cache.find_kept_packages(options.dry_run)
return UnattendedUpgradesResult(
kernel_pkgs_remove_success,
_("No packages found that can be upgraded unattended and no "
"pending auto-removals"),
pkgs_removed=kernel_pkgs_removed,
pkgs_kept_back=pkgs_kept_back,
pkgs_kept_installed=kernel_pkgs_kept_installed,
update_stamp=True)
# check if its configured for install on shutdown, if so, the
# environment UNATTENDED_UPGRADES_FORCE_INSTALL_ON_SHUTDOWN will
# be set by the unatteded-upgrades-shutdown script
if ("UNATTENDED_UPGRADES_FORCE_INSTALL_ON_SHUTDOWN" not in os.environ
and apt_pkg.config.find_b(
"Unattended-Upgrade::InstallOnShutdown", False)):
logger = logging.getLogger()
logger.debug("Configured to install on shutdown, so exiting now")
return UnattendedUpgradesResult(True)
# check if we are in dry-run mode
if options.dry_run:
logging.info("Option --dry-run given, *not* performing real actions")
apt_pkg.config.set("Debug::pkgDPkgPM", "1")
# do the install based on the new list of pkgs
pkgs = [pkg.name for pkg in pkgs_to_upgrade]
logging.info(_("Packages that will be upgraded: %s"), " ".join(pkgs))
# only perform install step if we actually have packages to install
pkg_install_success = True
if len(pkgs_to_upgrade) > 0:
# do install
pkg_install_success = do_install(cache,
pkgs,
options,
logfile_dpkg)
# Was the overall run succesful: only if everything installed
# fine and nothing was held back because of a conffile prompt.
successful_run = (kernel_pkgs_remove_success and pkg_install_success
and not pkg_conffile_prompt)
# now check if any auto-removing needs to be done
if cache._depcache.broken_count > 0:
print(_("Cache has broken packages, exiting"))
logging.error(_("Cache has broken packages, exiting"))
return UnattendedUpgradesResult(
False, _("Cache has broken packages, exiting"), pkgs=pkgs)
# make sure we start autoremovals with a clear cache
if cache.get_changes():
cache.clear()
# the user wants *all* auto-removals to be removed
# (unless u-u got signalled to stop gracefully quickly)
pkgs_removed = [] # type: List[str]
pkgs_kept_installed = [] # type: List[str]
if ((apt_pkg.config.find_b(
"Unattended-Upgrade::Remove-Unused-Dependencies", False)
and not SIGNAL_STOP_REQUEST)):
auto_removals = get_auto_removable(cache)
(pkg_remove_success,
pkgs_removed,
pkgs_kept_installed) = do_auto_remove(
cache, auto_removals, logfile_dpkg, options.minimal_upgrade_steps,
options.verbose or options.debug,
options.dry_run)
successful_run = successful_run and pkg_remove_success
# the user wants *only new* auto-removals to be removed
elif apt_pkg.config.find_b(
"Unattended-Upgrade::Remove-New-Unused-Dependencies", True):
# calculate the new auto-removals
new_pending_autoremovals = get_auto_removable(cache)
auto_removals = new_pending_autoremovals - previous_autoremovals
(pkg_remove_success,
pkgs_removed,
pkgs_kept_installed) = do_auto_remove(
cache, auto_removals, logfile_dpkg, options.minimal_upgrade_steps,
options.verbose or options.debug,
options.dry_run)
successful_run = successful_run and pkg_remove_success
logging.debug("InstCount=%i DelCount=%i BrokenCount=%i"
% (cache._depcache.inst_count,
cache._depcache.del_count,
cache._depcache.broken_count))
# clean after success install (if needed)
keep_key = "Unattended-Upgrade::Keep-Debs-After-Install"
if (not apt_pkg.config.find_b(keep_key, False)
and not options.dry_run
and pkg_install_success):
clean_downloaded_packages(fetcher)
pkgs_kept_back = cache.find_kept_packages(options.dry_run)
return UnattendedUpgradesResult(
successful_run, _("All upgrades installed"), pkgs,
pkgs_kept_back,
kernel_pkgs_removed + pkgs_removed,
kernel_pkgs_kept_installed + pkgs_kept_installed,
update_stamp=True)
class Options:
def __init__(self):
self.download_only = False
self.dry_run = False
self.debug = False
self.apt_debug = False
self.verbose = False
self.minimal_upgrade_steps = False
if __name__ == "__main__":
localesApp = "unattended-upgrades"
localesDir = "/usr/share/locale"
gettext.bindtextdomain(localesApp, localesDir)
gettext.textdomain(localesApp)
# set debconf to NON_INTERACTIVE
os.environ["DEBIAN_FRONTEND"] = "noninteractive"
# this ensures the commandline is logged in /var/log/apt/history.log
apt_pkg.config.set("Commandline::AsString", " ".join(sys.argv))
# COMPAT with the mispelling
minimal_steps_default = (
apt_pkg.config.find_b("Unattended-Upgrades::MinimalSteps", True)
and apt_pkg.config.find_b("Unattended-Upgrade::MinimalSteps", True))
# init the options
parser = OptionParser()
parser.add_option("-d", "--debug",
action="store_true",
default=apt_pkg.config.find_b(
"Unattended-Upgrade::Debug", False),
help=_("print debug messages"))
parser.add_option("", "--apt-debug",
action="store_true", default=False,
help=_("make apt/libapt print verbose debug messages"))
parser.add_option("-v", "--verbose",
action="store_true",
default=apt_pkg.config.find_b(
"Unattended-Upgrade::Verbose", False),
help=_("print info messages"))
parser.add_option("", "--dry-run",
action="store_true", default=False,
help=_("Simulation, download but do not install"))
parser.add_option("", "--download-only",
action="store_true", default=False,
help=_("Only download, do not even try to install."))
parser.add_option("", "--minimal-upgrade-steps",
action="store_true", default=minimal_steps_default,
help=_("Upgrade in minimal steps (and allow "
"interrupting with SIGTERM) (default)"))
parser.add_option("", "--no-minimal-upgrade-steps",
action="store_false", default=minimal_steps_default,
dest="minimal_upgrade_steps",
help=_("Upgrade all packages together instead of in "
"smaller sets"))
parser.add_option("", "--minimal_upgrade_steps",
action="store_true",
help=SUPPRESS_HELP,
default=minimal_steps_default)
options = cast(Options, (parser.parse_args())[0])
if os.getuid() != 0:
print(_("You need to be root to run this application"))
sys.exit(1)
# ensure that we are not killed when the terminal goes away e.g. on
# shutdown
signal.signal(signal.SIGHUP, signal.SIG_IGN)
# setup signal handler for graceful stopping
signal.signal(signal.SIGTERM, signal_handler)
# write pid to let other processes find this one
pidf = os.path.join(apt_pkg.config.find_dir("Dir"),
"var", "run", "unattended-upgrades.pid")
# clean up pid file on exit
with open(pidf, "w") as fp:
fp.write("%s" % os.getpid())
atexit.register(os.remove, pidf)
# run the main code
sys.exit(main(options))
Mini Shell