Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
beebeeep committed Dec 28, 2017
2 parents dcc000c + 429a3a9 commit 51de5cf
Show file tree
Hide file tree
Showing 20 changed files with 295 additions and 122 deletions.
3 changes: 3 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"python.pythonPath": "${workspaceRoot}/venv/bin/python3.6"
}
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## [Version 0.8.1](https://github.com/beebeeep/cacus/tree/v0.8.1) (2017-11-17)
* Migrated to python3, removed some libs poorly supported
* Fixed distro settings update via api/v1/distro/create

## [Version 0.7.25](https://github.com/beebeeep/cacus/tree/v0.7.25) (2017-12-13)
* Do not reindex repo while applying retention policy

Expand Down
19 changes: 9 additions & 10 deletions cacus/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
class MyLogger(logging.getLoggerClass()):
user = None

def makeRecord(self, name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None):
def makeRecord(self, name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None):
if not extra:
extra = {}
extra['user'] = self.user or 'N/A'
return super(MyLogger, self).makeRecord(name, lvl, fn, lno, msg, args, exc_info, func, extra)
return super(MyLogger, self).makeRecord(name, lvl, fn, lno, msg, args, exc_info, func=func, extra=extra, sinfo=None)

def error(self, msg, *args, **kwargs):
if sys.exc_info()[0]:
Expand All @@ -30,11 +30,10 @@ def critical(self, msg, *args, **kwargs):

logging.setLoggerClass(MyLogger)

import common
import repo_manage
import repo_daemon
import duploader
import distro_import
from . import repo_manage
from . import repo_daemon
from . import duploader
from . import distro_import

env_choices = ['unstable', 'testing', 'prestable', 'stable']

Expand Down Expand Up @@ -98,9 +97,9 @@ def main():

manager = repo_manage.RepoManager(config_file=args.config, quiet=True)
token = manager.generate_token(args.gen_token, args.expire, args.distro)
print "Generated token for '{}' with {}; valid for {} days:\n{}".format(
print("Generated token for '{}' with {}; valid for {} days:\n{}".format(
args.gen_token, 'access to distros: ' + ', '.join(args.distro) if args.distro else 'ROOT access',
args.expire, token)
args.expire, token))
elif args.revoke_token:
manager = repo_manage.RepoManager(config_file=args.config, quiet=True)
if manager.revoke_token(args.revoke_token):
Expand All @@ -112,7 +111,7 @@ def main():
manager.print_tokens()
elif args.get_token:
manager = repo_manage.RepoManager(config_file=args.config, quiet=True)
print manager.get_token(args.get_token)
print(manager.get_token(args.get_token))
else:
# default action is to start both duploader daemon and repo daemon
from multiprocessing import Process
Expand Down
37 changes: 5 additions & 32 deletions cacus/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import hashlib
import requests
import logging
import apt_pkg
import logging.handlers

from jose import jwt
Expand All @@ -25,7 +24,7 @@
from ipaddress import ip_network
from pymongo.collection import ReturnDocument

import plugin
from . import plugin

consul = None # optional module

Expand Down Expand Up @@ -79,32 +78,6 @@ class DistroLockTimeout(CacusError):
http_code = 409


class DebVersion(object):
""" Just wrapper around apt_pkg.version_compare() """

def __init__(self, version):
apt_pkg.init_system()
self.version = version

def __eq__(self, x):
return apt_pkg.version_compare(self.version, x.version) == 0

def __ne__(self, x):
return not self == x

def __lt__(self, x):
return apt_pkg.version_compare(self.version, x.version) < 0

def __gt__(self, x):
return apt_pkg.version_compare(self.version, x.version) > 0

def __ge__(self, x):
return self == x or self > x

def __le__(self, x):
return self == x or self < x


class Cacus(object):
default_arches = ['all', 'amd64', 'i386']
admin_access = '#admin'
Expand Down Expand Up @@ -188,7 +161,7 @@ def filter(self, record):
# misc
self.config['repo_daemon']['repo_base'] = self.config['repo_daemon']['repo_base'].rstrip('/')
self.config['repo_daemon']['storage_subdir'] = self.config['repo_daemon']['storage_subdir'].rstrip('/').lstrip('/')
self.config['repo_daemon']['privileged_nets'] = [ip_network(unicode(x)) for x in self.config['repo_daemon']['privileged_nets']]
self.config['repo_daemon']['privileged_nets'] = [ip_network(x) for x in self.config['repo_daemon']['privileged_nets']]

@staticmethod
def load_config(config_file):
Expand Down Expand Up @@ -313,7 +286,7 @@ def get_hashes(file=None, filename=None):
# XXX: This is pretty fat function, but I have no idea how to optimize it - my tests shows that
# it's almost as fast as "openssl [md5,sha1,sha256,sha256]", despite it's pretty straightforward approach
if filename:
file = open(filename)
file = open(filename, "rb")

md5 = hashlib.md5()
sha1 = hashlib.sha1()
Expand Down Expand Up @@ -350,7 +323,7 @@ def download_file(self, url, filename=None, md5=None, sha1=None, sha256=None):
_md5 = hashlib.md5()
_sha1 = hashlib.sha1()
_sha256 = hashlib.sha256()
with open(filename, 'w') as f:
with open(filename, 'wb') as f:
for chunk in r.iter_content(4*1024*1024):
total_bytes += len(chunk)
f.write(chunk)
Expand Down Expand Up @@ -590,7 +563,7 @@ def with_retries(attempts, delays, fun, *args, **kwargs):
# repeat last delay infinitely
delays = chain(delays[:-1], repeat(delays[-1]))
exc = Exception("Don't blink!")
for attempt in xrange(attempts):
for attempt in range(attempts):
try:
result = fun(*args, **kwargs)
except (Timeout, TemporaryError, DistroLockTimeout) as e:
Expand Down
2 changes: 1 addition & 1 deletion cacus/distro_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

from debian import deb822

import repo_manage
from . import repo_manage


class DistroImporter(repo_manage.RepoManager):
Expand Down
10 changes: 5 additions & 5 deletions cacus/duploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
import signal
import threading
import time
import Queue
import queue

import pyinotify
from debian import deb822

import common
import repo_manage
from . import common
from . import repo_manage


class ComplexDistroWatcher(pyinotify.ProcessEvent):
Expand Down Expand Up @@ -166,7 +166,7 @@ def __init__(self, repo_manager, settings, component):
self.distro = settings['distro']
self.component = component
self.log = repo_manager.log
self.queue = Queue.Queue()
self.queue = queue.Queue()
self.worker = threading.Thread(target=self._worker)
self.worker.daemon = True
self.worker.start()
Expand Down Expand Up @@ -238,7 +238,7 @@ def run(self):
try:
if not os.path.isdir(incoming_dir):
self.log.debug("Creating incoming dir '%s'", incoming_dir)
os.makedirs(incoming_dir, mode=0777)
os.makedirs(incoming_dir, mode=0o777)
except Exception as e:
self.log.error("Cannot create dir for incoming files '%s': %s", incoming_dir, e)
continue
Expand Down
134 changes: 134 additions & 0 deletions cacus/extras.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#!/usr/bin/env python3

import re


class DebVersion(object):
""" Implements debian package version comparsion according to deb-version(5)
I'm not responsible for those brain-dead people who invented this sick format and comparsion rules
Same functionality is provided by python-apt, but it's poorly maintained
"""

__version_re = re.compile(r'''^((?P<epoch>\d+):)?(?P<upstream>[-.+:~0-9A-z]+?)(-(?P<debian>[+.~0-9A-z]+))?$''')
__digits_re = re.compile(r'''^(\d+)''')
__nondigits_re = re.compile(r'''^([-.+:~A-z]+)''')

# table for wicked debianish string comparsion
__lexical_table = {
'~': 0, '': 1, '+': 2, '-': 3, '.': 4, ':': 5
}
__lexical_table.update(dict((chr(x), x) for x in range(ord('A'), ord('z'))))

def __init__(self, version):
self.version = version.rstrip().lstrip()
m = self.__version_re.match(self.version)
if m:
self.correct = True
g = m.groupdict()
self.epoch = int(g['epoch'] or 0)
self.upstream = g['upstream']
self.debian = g['debian']
else:
self.correct = False
self.epoch = self.upstream = self.debian = None

def __part_lt(self, x1, x2):
""" returns True if x1 < x2 according to string comparsion rules defined in section "Sorting algorithm" of deb-version(5)
"""
while True:
if not x1 and not x2:
# both strings exhausted, exit
return False
elif not x1:
# otherwise complement None-s to empty string as emptiness matters
x1 = ''
elif not x2:
x2 = ''

# extract non-digit part from both strings
m1 = self.__nondigits_re.search(x1)
m2 = self.__nondigits_re.search(x2)
s1 = m1.group(1) if m1 else ''
s2 = m2.group(1) if m2 else ''
len_s1 = len(s1)
len_s2 = len(s2)
i = -1
while True:
# compare both parts
i = i + 1
c1 = s1[i] if i < len_s1 else ''
c2 = s2[i] if i < len_s2 else ''
if c1 == '' and c2 == '':
break
if c1 == c2:
continue
if self.__lexical_table[c1] < self.__lexical_table[c2]:
return True
if self.__lexical_table[c1] > self.__lexical_table[c2]:
return False

# cut analyzed parts from string
x1 = x1[len_s1:]
x2 = x2[len_s2:]

m1 = self.__digits_re.search(x1)
m2 = self.__digits_re.search(x2)
s1 = m1.group(1) if m1 else ''
s2 = m2.group(1) if m2 else ''
number1 = int(s1 or 0)
number2 = int(s2 or 0)
if number1 < number2:
return True
elif number1 > number2:
return False

# cut analyzed numbers from string and go over
x1 = x1[len(s1):]
x2 = x2[len(s2):]

def __eq__(self, x):
return self.__cmp__(x) == 0

def __ne__(self, x):
return self.__cmp__(x) != 0

def __lt__(self, x):
return self.__cmp__(x) < 0

def __gt__(self, x):
return self.__cmp__(x) > 0

def __ge__(self, x):
return self.__cmp__(x) >= 0

def __le__(self, x):
return self.__cmp__(x) <= 0

def __cmp__(self, x):

if self.epoch == x.epoch and self.upstream == x.upstream and self.debian == x.debian:
return 0

if not self.correct:
# incorrect versions are always lesser
return -1

if self.epoch != x.epoch:
if self.epoch < x.epoch:
return -1
else:
return 1

if self.upstream != x.upstream:
if self.__part_lt(self.upstream, x.upstream):
return -1
else:
return 1

if self.debian != x.debian:
if self.__part_lt(self.debian, x.debian):
return -1
else:
return 1

return 0
Loading

0 comments on commit 51de5cf

Please sign in to comment.