Loading CHANGELOG +3 −1 Changes for CHANGELOG: 3 added lines, 1 removed line. Original line number Diff line number Diff line This project currently does not maintain a changelog file. No newline at end of file V 1.1.0: - Rewrote the program to use oauth for cloning - Program now retrieves all projects accesible to the private token No newline at end of file README.md +4 −5 Changes for README.md: 4 added lines, 5 removed lines. Original line number Diff line number Diff line # Gitlab Cloner [](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/master) |master|develop| |:---:|:---:| |[](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/master)|[](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/develop)|  This is a small script that clones all git repositories of a user from a This is a program that clones all git repositories of a user from a Gitlab server. This requires the user's API access token. Loading @@ -17,8 +18,6 @@ one should be the API key. You can also specify a destination directory for the repositories with the ```-d``` argument. To include any archived repositories, use the ```-a``` option ## Further Information * [Changelog](CHANGELOG) Loading bin/gitlab-cloner +6 −104 Changes for bin/gitlab-cloner: 6 added lines, 104 removed lines. Original line number Diff line number Diff line Loading @@ -18,121 +18,23 @@ You should have received a copy of the GNU General Public License along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>. LICENSE""" from __future__ import print_function import os import sys import json import argparse import requests from subprocess import Popen, PIPE from gitlab_cloner import clone_all def main(): """ The main function of this script Retrieves a list of all projects in a gitlab instance and clones them :return: None """ args = parse_args() projects = get_repositories(args.url, args.user, args.token, args.archived) clone_repos(projects, args.destination) def parse_args(): """ Parses the Command Line Arguments """ parser = argparse.ArgumentParser() parser.add_argument("url", help="The Gitlab URL") parser.add_argument("user", help="The Gitlab user") parser.add_argument("token", help="The Gitlab API Token") parser.add_argument("-a", "--archived", default=False, action="store_true", help="Includes archived repositories") parser.add_argument("url", help="The Gitlab URL, including https etc") parser.add_argument("token", help="The Gitlab Personal Access Token") parser.add_argument("-d", "--destination", default=os.getcwd(), help="The destination directory.") return parser.parse_args() def clone_repos(projects, destination): """ Clones the Repositories to the destination directory """ destination = os.path.abspath(destination) if not os.path.exists(destination): os.makedirs(destination) if not os.path.isdir(destination): print("Could not create directory " + destination) sys.exit(1) cwd = os.getcwd() # Remember the current working directory for project in projects: os.chdir(destination) name = project["name"] repo_destination = os.path.join(destination, name) if os.path.isdir(repo_destination): # If the repository has been cloned already, just fetch new changes print("Fetching " + name, end="... ") command = ["git", "fetch", "--all"] os.chdir(repo_destination) else: print("Cloning " + name, end="... ") ssh_url = project["ssh_url_to_repo"] command = ["git", "clone", ssh_url] process = Popen(command, stdout=PIPE, stderr=PIPE) process.communicate() print("Done.") os.chdir(cwd) # Return to original working directory def get_repositories(gitlab_url, user, api_key, archived): """ Fetches the repository information from the gitlab API. Requires a valid Gitlab URL, the user's access token. If archived is True, also include archived repositories """ if not gitlab_url.endswith("/"): gitlab_url += "/" query = gitlab_url + "api/v4/users/" + user + "/projects" query += "?private_token=" + api_key if archived: query += "&archived=true" result = requests.get(query) projects = [] count = 1 more_pages = True while more_pages: more_pages = result.headers["X-Next-Page"] != "" if result.status_code != 200: # If unauthorized or other error, stop. print(result.text) sys.exit(1) else: projects += json.loads(result.text) count += 1 result = requests.get(query + "&page=" + str(count)) return projects args = parser.parse_args() clone_all(args.url, args.token, args.destination) if __name__ == "__main__": Loading gitlab_cloner/__init__.py +25 −0 Changes for gitlab_cloner/__init__.py: 25 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -16,3 +16,28 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>. LICENSE""" import os from gitlab_cloner.gitlab_api import clone_repo, get_projects def clone_all(url: str, token: str, destination: str): """ Clones all repositories accessible to the personal access token provided :param url: The gitlab URL, including https :param token: The personal access token to use :param destination: The destination in which to save the repos :return: None """ current = os.getcwd() if not os.path.isdir(destination): os.makedirs(destination) os.chdir(destination) projects = get_projects(url, token) for project in projects: clone_repo(project, token) os.chdir(current) gitlab_cloner/gitlab_api.py 0 → 100644 +78 −0 Changes for gitlab_cloner/gitlab_api.py: 78 added lines, 0 removed lines. Original line number Diff line number Diff line """LICENSE Copyright 2017 Hermann Krumrey <hermann@krumreyh.com> This file is part of gitlab-cloner. gitlab-cloner is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. gitlab-cloner is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>. LICENSE""" import os import json import shutil import requests from typing import List, Dict, Any from subprocess import check_output from colorama import Fore, Style def get_projects(gitlab_url: str, private_token: str) \ -> List[Dict[str, Any]]: """ Retrieves a list of repositories that the provided private token grants access to. :param gitlab_url: The URL of the gitlab instance, including https etc. :param private_token: The private token to use :return: A list of gitlab projects """ url = gitlab_url + "/api/v4/projects?private_token=" + private_token response = requests.get(url) projects = json.loads(response.text) while response.headers["X-Next-Page"]: next_page = response.headers["X-Next-Page"] response = requests.get(url + "&page=" + next_page) projects += json.loads(response.text) return projects def clone_repo(repo: Dict[str, Any], private_token: str): """ Clones a git repository. If the repository already exists, the user will be prompted to delete the old repository. If the user refuses, the repository will not be cloned. :param repo: The repository to clone :param private_token: The private token to use when cloning :return: None """ if os.path.isdir(repo["name"]): resp = input("Repository " + repo["name"] + "exists. Delete? (y|n)") if resp == "y": shutil.rmtree(repo["name"]) else: return oauth = "oauth2:" + private_token + "@" url = repo["http_url_to_repo"]\ .replace("http://", "http://" + oauth)\ .replace("https://", "https://" + oauth) command = ["git", "clone", url] print(Fore.CYAN + " ".join(command) + Style.RESET_ALL) output = check_output(command) print(Fore.MAGENTA + output.decode() + Style.RESET_ALL) Loading
CHANGELOG +3 −1 Changes for CHANGELOG: 3 added lines, 1 removed line. Original line number Diff line number Diff line This project currently does not maintain a changelog file. No newline at end of file V 1.1.0: - Rewrote the program to use oauth for cloning - Program now retrieves all projects accesible to the private token No newline at end of file
README.md +4 −5 Changes for README.md: 4 added lines, 5 removed lines. Original line number Diff line number Diff line # Gitlab Cloner [](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/master) |master|develop| |:---:|:---:| |[](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/master)|[](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/develop)|  This is a small script that clones all git repositories of a user from a This is a program that clones all git repositories of a user from a Gitlab server. This requires the user's API access token. Loading @@ -17,8 +18,6 @@ one should be the API key. You can also specify a destination directory for the repositories with the ```-d``` argument. To include any archived repositories, use the ```-a``` option ## Further Information * [Changelog](CHANGELOG) Loading
bin/gitlab-cloner +6 −104 Changes for bin/gitlab-cloner: 6 added lines, 104 removed lines. Original line number Diff line number Diff line Loading @@ -18,121 +18,23 @@ You should have received a copy of the GNU General Public License along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>. LICENSE""" from __future__ import print_function import os import sys import json import argparse import requests from subprocess import Popen, PIPE from gitlab_cloner import clone_all def main(): """ The main function of this script Retrieves a list of all projects in a gitlab instance and clones them :return: None """ args = parse_args() projects = get_repositories(args.url, args.user, args.token, args.archived) clone_repos(projects, args.destination) def parse_args(): """ Parses the Command Line Arguments """ parser = argparse.ArgumentParser() parser.add_argument("url", help="The Gitlab URL") parser.add_argument("user", help="The Gitlab user") parser.add_argument("token", help="The Gitlab API Token") parser.add_argument("-a", "--archived", default=False, action="store_true", help="Includes archived repositories") parser.add_argument("url", help="The Gitlab URL, including https etc") parser.add_argument("token", help="The Gitlab Personal Access Token") parser.add_argument("-d", "--destination", default=os.getcwd(), help="The destination directory.") return parser.parse_args() def clone_repos(projects, destination): """ Clones the Repositories to the destination directory """ destination = os.path.abspath(destination) if not os.path.exists(destination): os.makedirs(destination) if not os.path.isdir(destination): print("Could not create directory " + destination) sys.exit(1) cwd = os.getcwd() # Remember the current working directory for project in projects: os.chdir(destination) name = project["name"] repo_destination = os.path.join(destination, name) if os.path.isdir(repo_destination): # If the repository has been cloned already, just fetch new changes print("Fetching " + name, end="... ") command = ["git", "fetch", "--all"] os.chdir(repo_destination) else: print("Cloning " + name, end="... ") ssh_url = project["ssh_url_to_repo"] command = ["git", "clone", ssh_url] process = Popen(command, stdout=PIPE, stderr=PIPE) process.communicate() print("Done.") os.chdir(cwd) # Return to original working directory def get_repositories(gitlab_url, user, api_key, archived): """ Fetches the repository information from the gitlab API. Requires a valid Gitlab URL, the user's access token. If archived is True, also include archived repositories """ if not gitlab_url.endswith("/"): gitlab_url += "/" query = gitlab_url + "api/v4/users/" + user + "/projects" query += "?private_token=" + api_key if archived: query += "&archived=true" result = requests.get(query) projects = [] count = 1 more_pages = True while more_pages: more_pages = result.headers["X-Next-Page"] != "" if result.status_code != 200: # If unauthorized or other error, stop. print(result.text) sys.exit(1) else: projects += json.loads(result.text) count += 1 result = requests.get(query + "&page=" + str(count)) return projects args = parser.parse_args() clone_all(args.url, args.token, args.destination) if __name__ == "__main__": Loading
gitlab_cloner/__init__.py +25 −0 Changes for gitlab_cloner/__init__.py: 25 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -16,3 +16,28 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>. LICENSE""" import os from gitlab_cloner.gitlab_api import clone_repo, get_projects def clone_all(url: str, token: str, destination: str): """ Clones all repositories accessible to the personal access token provided :param url: The gitlab URL, including https :param token: The personal access token to use :param destination: The destination in which to save the repos :return: None """ current = os.getcwd() if not os.path.isdir(destination): os.makedirs(destination) os.chdir(destination) projects = get_projects(url, token) for project in projects: clone_repo(project, token) os.chdir(current)
gitlab_cloner/gitlab_api.py 0 → 100644 +78 −0 Changes for gitlab_cloner/gitlab_api.py: 78 added lines, 0 removed lines. Original line number Diff line number Diff line """LICENSE Copyright 2017 Hermann Krumrey <hermann@krumreyh.com> This file is part of gitlab-cloner. gitlab-cloner is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. gitlab-cloner is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>. LICENSE""" import os import json import shutil import requests from typing import List, Dict, Any from subprocess import check_output from colorama import Fore, Style def get_projects(gitlab_url: str, private_token: str) \ -> List[Dict[str, Any]]: """ Retrieves a list of repositories that the provided private token grants access to. :param gitlab_url: The URL of the gitlab instance, including https etc. :param private_token: The private token to use :return: A list of gitlab projects """ url = gitlab_url + "/api/v4/projects?private_token=" + private_token response = requests.get(url) projects = json.loads(response.text) while response.headers["X-Next-Page"]: next_page = response.headers["X-Next-Page"] response = requests.get(url + "&page=" + next_page) projects += json.loads(response.text) return projects def clone_repo(repo: Dict[str, Any], private_token: str): """ Clones a git repository. If the repository already exists, the user will be prompted to delete the old repository. If the user refuses, the repository will not be cloned. :param repo: The repository to clone :param private_token: The private token to use when cloning :return: None """ if os.path.isdir(repo["name"]): resp = input("Repository " + repo["name"] + "exists. Delete? (y|n)") if resp == "y": shutil.rmtree(repo["name"]) else: return oauth = "oauth2:" + private_token + "@" url = repo["http_url_to_repo"]\ .replace("http://", "http://" + oauth)\ .replace("https://", "https://" + oauth) command = ["git", "clone", url] print(Fore.CYAN + " ".join(command) + Style.RESET_ALL) output = check_output(command) print(Fore.MAGENTA + output.decode() + Style.RESET_ALL)
