backup_gitlab.com/backup_gitlab.py at master · poweredbygrow/backup_gitlab.com · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/usr/bin/env python3

import datetime
import logging
import os
import requests
import shutil
import subprocess
import sys
import time

ROOT = os.path.dirname(os.path.realpath(__file__))

import conf

os.makedirs(os.path.join(ROOT, 'log'), exist_ok=True)
logging.basicConfig(filename=datetime.datetime.now().strftime(os.path.join(ROOT, "log/gitlabbackup_%Y_%m_%d_%H_%M.log")), level=logging.INFO)
logger = logging.getLogger('gitlab-backup')


def clean(dir_to_clean):
    if os.path.isdir(dir_to_clean):
        shutil.rmtree(dir_to_clean)


def run(cmd):
    logger.info("running:" + ' '.join(cmd))
    subprocess.check_call(cmd)


def get_projects():
    projects = []
    page = 1
    while True:
        params = {'private_token': conf.private_token,
                  'membership': 'yes',
                  'per_page': 50,
                  'page': page,
                  'order_by': 'name',
                  'sort': 'asc'}
        r = requests.get('https://gitlab.com/api/v4/projects', params=params)
        if r.status_code != 200:
            raise Exception("Didn't get a 200. Code: " + str(r.status_code) + "\nBody:\n" + r.text)
        new_projects = r.json()
        names = [project['name'] for project in new_projects]
        logger.info('Got: ' + ' '.join(names))
        if len(new_projects) == 0:
            break
        page += 1
        projects += new_projects

    return projects


def mirror_git_repo(http_url, repo_dir):
    try:
        run(['git', 'clone', '--mirror', http_url])
    except KeyboardInterrupt:
        clean(repo_dir)
        sys.exit(1)
    except Exception:
        clean(repo_dir)


def update_git_repo(repo_dir):
    try:
        old_dir = os.getcwd()
        os.chdir(repo_dir)
        run(['git', 'remote', 'update'])
    finally:
        os.chdir(old_dir)


def backup_gitlab():
    start = time.time()
    os.makedirs(conf.backup_dir, exist_ok=True)
    os.chdir(conf.backup_dir)

    projects = get_projects()
    print('Found ', len(projects), 'projects')
    for project in projects:
        print('*'*80)
        id = project['id']
        web_url = project['web_url']
        print('id:', id, 'web_url:', web_url)

        http_url = project['ssh_url_to_repo']
        # Remove the entire url except the last part which is
        # what the mirrored directory will be called
        repo_dir = http_url.split('/')[-1]

        if os.path.isdir(repo_dir):
            try:
                update_git_repo(repo_dir)
            except subprocess.CalledProcessError:
                logger.info('Updating repo_dir failed. Trying to delete and mirror')
                shutil.rmtree(repo_dir)
                mirror_git_repo(http_url, repo_dir)

        else:
            mirror_git_repo(http_url, repo_dir)
    end = time.time()
    logger.info('took: ' + str(end-start) + 's')


def main():
    backup_gitlab()


if __name__ == '__main__':
    main()