Commit 984af6a3 authored by Hermann Krumrey's avatar Hermann Krumrey

Rewrite project to work with different namespaces

parent 0f767399
This project currently does not maintain a changelog file.
\ No newline at end of file
V 1.1.0:
- Rewrote the program to use oauth for cloning
- Program now retrieves all projects accesible to the private token
\ No newline at end of file
# Gitlab Cloner
[![build status](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/badges/master/build.svg)](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/master)
|master|develop|
|:---:|:---:|
|[![build status](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/badges/master/build.svg)](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/master)|[![build status](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/badges/develop/build.svg)](https://gitlab.namibsun.net/namibsun/python/gitlab-cloner/commits/develop)|
![Logo](resources/logo/logo-readme.png "Logo")
This is a small script that clones all git repositories of a user from a
This is a program that clones all git repositories of a user from a
Gitlab server.
This requires the user's API access token.
......@@ -17,8 +18,6 @@ one should be the API key.
You can also specify a destination directory for the repositories
with the ```-d``` argument.
To include any archived repositories, use the ```-a``` option
## Further Information
* [Changelog](CHANGELOG)
......
......@@ -18,121 +18,23 @@ You should have received a copy of the GNU General Public License
along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
from __future__ import print_function
import os
import sys
import json
import argparse
import requests
from subprocess import Popen, PIPE
from gitlab_cloner import clone_all
def main():
"""
The main function of this script
Retrieves a list of all projects in a gitlab instance and clones them
:return: None
"""
args = parse_args()
projects = get_repositories(args.url, args.user, args.token, args.archived)
clone_repos(projects, args.destination)
def parse_args():
"""
Parses the Command Line Arguments
"""
parser = argparse.ArgumentParser()
parser.add_argument("url", help="The Gitlab URL")
parser.add_argument("user", help="The Gitlab user")
parser.add_argument("token", help="The Gitlab API Token")
parser.add_argument("-a", "--archived", default=False, action="store_true",
help="Includes archived repositories")
parser.add_argument("url", help="The Gitlab URL, including https etc")
parser.add_argument("token", help="The Gitlab Personal Access Token")
parser.add_argument("-d", "--destination", default=os.getcwd(),
help="The destination directory.")
return parser.parse_args()
def clone_repos(projects, destination):
"""
Clones the Repositories to the destination directory
"""
destination = os.path.abspath(destination)
if not os.path.exists(destination):
os.makedirs(destination)
if not os.path.isdir(destination):
print("Could not create directory " + destination)
sys.exit(1)
cwd = os.getcwd() # Remember the current working directory
for project in projects:
os.chdir(destination)
name = project["name"]
repo_destination = os.path.join(destination, name)
if os.path.isdir(repo_destination):
# If the repository has been cloned already, just fetch new changes
print("Fetching " + name, end="... ")
command = ["git", "fetch", "--all"]
os.chdir(repo_destination)
else:
print("Cloning " + name, end="... ")
ssh_url = project["ssh_url_to_repo"]
command = ["git", "clone", ssh_url]
process = Popen(command, stdout=PIPE, stderr=PIPE)
process.communicate()
print("Done.")
os.chdir(cwd) # Return to original working directory
def get_repositories(gitlab_url, user, api_key, archived):
"""
Fetches the repository information from the gitlab API.
Requires a valid Gitlab URL, the user's access token.
If archived is True, also include archived repositories
"""
if not gitlab_url.endswith("/"):
gitlab_url += "/"
query = gitlab_url + "api/v4/users/" + user + "/projects"
query += "?private_token=" + api_key
if archived:
query += "&archived=true"
result = requests.get(query)
projects = []
count = 1
more_pages = True
while more_pages:
more_pages = result.headers["X-Next-Page"] != ""
if result.status_code != 200: # If unauthorized or other error, stop.
print(result.text)
sys.exit(1)
else:
projects += json.loads(result.text)
count += 1
result = requests.get(query + "&page=" + str(count))
return projects
args = parser.parse_args()
clone_all(args.url, args.token, args.destination)
if __name__ == "__main__":
......
......@@ -16,3 +16,28 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
import os
from gitlab_cloner.gitlab_api import clone_repo, get_projects
def clone_all(url: str, token: str, destination: str):
"""
Clones all repositories accessible to the personal access token provided
:param url: The gitlab URL, including https
:param token: The personal access token to use
:param destination: The destination in which to save the repos
:return: None
"""
current = os.getcwd()
if not os.path.isdir(destination):
os.makedirs(destination)
os.chdir(destination)
projects = get_projects(url, token)
for project in projects:
clone_repo(project, token)
os.chdir(current)
"""LICENSE
Copyright 2017 Hermann Krumrey <hermann@krumreyh.com>
This file is part of gitlab-cloner.
gitlab-cloner is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
gitlab-cloner is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
import os
import json
import shutil
import requests
from typing import List, Dict, Any
from subprocess import check_output
from colorama import Fore, Style
def get_projects(gitlab_url: str, private_token: str) \
-> List[Dict[str, Any]]:
"""
Retrieves a list of repositories that the provided private token
grants access to.
:param gitlab_url: The URL of the gitlab instance, including https etc.
:param private_token: The private token to use
:return: A list of gitlab projects
"""
url = gitlab_url + "/api/v4/projects?private_token=" + private_token
response = requests.get(url)
projects = json.loads(response.text)
while response.headers["X-Next-Page"]:
next_page = response.headers["X-Next-Page"]
response = requests.get(url + "&page=" + next_page)
projects += json.loads(response.text)
return projects
def clone_repo(repo: Dict[str, Any], private_token: str):
"""
Clones a git repository. If the repository already exists,
the user will be prompted to delete the old repository.
If the user refuses, the repository will not be cloned.
:param repo: The repository to clone
:param private_token: The private token to use when cloning
:return: None
"""
if os.path.isdir(repo["name"]):
resp = input("Repository " + repo["name"] + "exists. Delete? (y|n)")
if resp == "y":
shutil.rmtree(repo["name"])
else:
return
oauth = "oauth2:" + private_token + "@"
url = repo["http_url_to_repo"]\
.replace("http://", "http://" + oauth)\
.replace("https://", "https://" + oauth)
command = ["git", "clone", url]
print(Fore.CYAN + " ".join(command) + Style.RESET_ALL)
output = check_output(command)
print(Fore.MAGENTA + output.decode() + Style.RESET_ALL)
......@@ -41,7 +41,7 @@ if __name__ == "__main__":
license="GNU GPL3",
packages=find_packages(),
scripts=list(map(lambda x: os.path.join("bin", x), os.listdir("bin"))),
install_requires=["requests"],
install_requires=["requests", "colorama", "typing"],
include_package_data=True,
zip_safe=False
)
"""LICENSE
Copyright 2017 Hermann Krumrey <hermann@krumreyh.com>
This file is part of gitlab-cloner.
gitlab-cloner is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
gitlab-cloner is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
"""LICENSE
Copyright 2017 Hermann Krumrey <hermann@krumreyh.com>
This file is part of gitlab-cloner.
gitlab-cloner is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
gitlab-cloner is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with gitlab-cloner. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
import os
import shutil
from unittest import TestCase
from gitlab_cloner import clone_all, get_projects
class CloningTest(TestCase):
"""
Class that tests the cloning of gitlab repositories
"""
testdir = "repotest"
"""
The directory in which the repos will be stored
"""
def setUp(self):
"""
Runs cleanup
:return: None
"""
self.cleanup()
def tearDown(self):
"""
Runs cleanup
:return: None
"""
self.cleanup()
def cleanup(self):
"""
Deletes the testing directory
:return: None
"""
if os.path.isdir(self.testdir):
shutil.rmtree(self.testdir)
def test_cloning(self):
url = "https://gitlab.namibsun.net"
token = "" # Will only fetch public repos
destination = self.testdir
current = os.getcwd()
clone_all(url, token, destination)
self.assertEquals(current, os.getcwd())
projects = get_projects(url, token)
self.assertEquals(len(projects), len(os.listdir(self.testdir)))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment