From d3c463782d08280c832e33ec2322ca0da8944e28 Mon Sep 17 00:00:00 2001 From: Markus Kaiser Date: Tue, 3 May 2022 17:09:49 +0200 Subject: [PATCH] Add a blacklist for repositories --- README.md | 23 ++++++++++++----------- src/config.json | 1 + src/helper.py | 12 ++++++++++-- src/repositoryForked.py | 6 +++++- src/repositorySource.py | 6 +++++- src/repositoryStared.py | 6 +++++- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9b9b793..a666540 100644 --- a/README.md +++ b/README.md @@ -35,31 +35,32 @@ Or you can download the source code and install it where ever you need | --- | --- | | `github.username` | Your Github Username | | `github.accesstoken` | Your Github Account's Personal Access Token | -| - | - | +| - | - | | `gitea.host` | Selfhosted Gitea URL without `/` at the end | | `gitea.accesstoken` | Your Personal Access Token | | `gitea.username` | Account User Name | | `gitea.gist.prefix` | Custom Prefix For Repository When Mirroring Gists | | `gitea.gist.surfix` | Custom Prefix For Repository When Mirroring Gists | -| - | - | -| `repomap` | Remap A Repository To Diff User | -| `gistsSource` | set to true to mirror all Gists Created By You| -| `gistsStared` | set to true to mirror all Gists Stared By You| -| `repositoryStared` | set to true to mirror all Repository Stared By You | -| `repositorySource` | set to true to mirror all Repository Created By You | -| `repositoryForked` | set to true to mirror all Repository Forked By You | +| - | - | +| `repomap` | Remap A Repository To Diff User | +| `blacklist` | A list of glob-expression on github full repository names to skip | +| `gistsSource` | set to true to mirror all Gists Created By You| +| `gistsStared` | set to true to mirror all Gists Stared By You| +| `repositoryStared` | set to true to mirror all Repository Stared By You | +| `repositorySource` | set to true to mirror all Repository Created By You | +| `repositoryForked` | set to true to mirror all Repository Forked By You | | - | - | | `local_cache.enabled` | Set to **true** to store all repostiory slugs from gitea as json | | `local_cache.file_path` | Custom Path to store json file | -> Local Cache can come handly when running this script via cron +> Local Cache can come handly when running this script via cron > which reduces api request your selfhosted gitea instance ### Run cmd & Wait $ python3 mirror.py - - + + ## Cron Setup 1. Run `crontab -e` 2. `mkdir $HOME/mirrorLogs -p` diff --git a/src/config.json b/src/config.json index 85335af..426a03c 100644 --- a/src/config.json +++ b/src/config.json @@ -14,6 +14,7 @@ } }, "repomap" : { }, + "blacklist" : [ ], "gistsSource" : true, "gistsStared" : true, "repositoryStared" : true, diff --git a/src/helper.py b/src/helper.py index d164131..d110a95 100644 --- a/src/helper.py +++ b/src/helper.py @@ -5,6 +5,7 @@ import requests import json import sys import os +import fnmatch from datetime import datetime giteaGetUserCache = dict() @@ -78,9 +79,9 @@ def giteaCreateRepo(data,isPrivate,isRepository): data["service"] = 'github' data["wiki"] = True data["auth_token"] = "{0}".format(config['github']['accesstoken']) - - + + jsonstring = json.dumps(data) r = session.post(giteaHost('repos/migrate'), data=jsonstring) @@ -201,3 +202,10 @@ def giteaGetAllUsersOrgs(type): loopCount += 1 return results + + +def isBlacklistedRepository(full_name): + for pattern in config['blacklist']: + if fnmatch.fnmatch(full_name, pattern): + return True + return False diff --git a/src/repositoryForked.py b/src/repositoryForked.py index b803048..0a4e47c 100644 --- a/src/repositoryForked.py +++ b/src/repositoryForked.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,6 +21,10 @@ def repositoryForked(): log('Forked Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositorySource.py b/src/repositorySource.py index 292c309..7f40a03 100644 --- a/src/repositorySource.py +++ b/src/repositorySource.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,6 +21,10 @@ def repositorySource(): log('Source Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositoryStared.py b/src/repositoryStared.py index ac72a44..ebe3513 100644 --- a/src/repositoryStared.py +++ b/src/repositoryStared.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -20,6 +20,10 @@ def repositoryStared(): log('⭐ Star\'ed Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo]