From e9b56957b7ac86d1d48db398ad8dfb07e9913532 Mon Sep 17 00:00:00 2001 From: Markus Kaiser Date: Tue, 3 May 2022 17:09:49 +0200 Subject: [PATCH 1/4] Add a blacklist for repositories --- README.md | 23 ++++++++++++----------- src/config.json | 1 + src/helper.py | 12 ++++++++++-- src/repositoryForked.py | 6 +++++- src/repositorySource.py | 6 +++++- src/repositoryStared.py | 6 +++++- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9b9b793..a666540 100644 --- a/README.md +++ b/README.md @@ -35,31 +35,32 @@ Or you can download the source code and install it where ever you need | --- | --- | | `github.username` | Your Github Username | | `github.accesstoken` | Your Github Account's Personal Access Token | -| - | - | +| - | - | | `gitea.host` | Selfhosted Gitea URL without `/` at the end | | `gitea.accesstoken` | Your Personal Access Token | | `gitea.username` | Account User Name | | `gitea.gist.prefix` | Custom Prefix For Repository When Mirroring Gists | | `gitea.gist.surfix` | Custom Prefix For Repository When Mirroring Gists | -| - | - | -| `repomap` | Remap A Repository To Diff User | -| `gistsSource` | set to true to mirror all Gists Created By You| -| `gistsStared` | set to true to mirror all Gists Stared By You| -| `repositoryStared` | set to true to mirror all Repository Stared By You | -| `repositorySource` | set to true to mirror all Repository Created By You | -| `repositoryForked` | set to true to mirror all Repository Forked By You | +| - | - | +| `repomap` | Remap A Repository To Diff User | +| `blacklist` | A list of glob-expression on github full repository names to skip | +| `gistsSource` | set to true to mirror all Gists Created By You| +| `gistsStared` | set to true to mirror all Gists Stared By You| +| `repositoryStared` | set to true to mirror all Repository Stared By You | +| `repositorySource` | set to true to mirror all Repository Created By You | +| `repositoryForked` | set to true to mirror all Repository Forked By You | | - | - | | `local_cache.enabled` | Set to **true** to store all repostiory slugs from gitea as json | | `local_cache.file_path` | Custom Path to store json file | -> Local Cache can come handly when running this script via cron +> Local Cache can come handly when running this script via cron > which reduces api request your selfhosted gitea instance ### Run cmd & Wait $ python3 mirror.py - - + + ## Cron Setup 1. Run `crontab -e` 2. `mkdir $HOME/mirrorLogs -p` diff --git a/src/config.json b/src/config.json index 85335af..426a03c 100644 --- a/src/config.json +++ b/src/config.json @@ -14,6 +14,7 @@ } }, "repomap" : { }, + "blacklist" : [ ], "gistsSource" : true, "gistsStared" : true, "repositoryStared" : true, diff --git a/src/helper.py b/src/helper.py index d164131..d110a95 100644 --- a/src/helper.py +++ b/src/helper.py @@ -5,6 +5,7 @@ import requests import json import sys import os +import fnmatch from datetime import datetime giteaGetUserCache = dict() @@ -78,9 +79,9 @@ def giteaCreateRepo(data,isPrivate,isRepository): data["service"] = 'github' data["wiki"] = True data["auth_token"] = "{0}".format(config['github']['accesstoken']) - - + + jsonstring = json.dumps(data) r = session.post(giteaHost('repos/migrate'), data=jsonstring) @@ -201,3 +202,10 @@ def giteaGetAllUsersOrgs(type): loopCount += 1 return results + + +def isBlacklistedRepository(full_name): + for pattern in config['blacklist']: + if fnmatch.fnmatch(full_name, pattern): + return True + return False diff --git a/src/repositoryForked.py b/src/repositoryForked.py index b803048..0a4e47c 100644 --- a/src/repositoryForked.py +++ b/src/repositoryForked.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,6 +21,10 @@ def repositoryForked(): log('Forked Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositorySource.py b/src/repositorySource.py index 292c309..7f40a03 100644 --- a/src/repositorySource.py +++ b/src/repositorySource.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,6 +21,10 @@ def repositorySource(): log('Source Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositoryStared.py b/src/repositoryStared.py index ac72a44..ebe3513 100644 --- a/src/repositoryStared.py +++ b/src/repositoryStared.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -20,6 +20,10 @@ def repositoryStared(): log('⭐ Star\'ed Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] From 8462047484c8d2b642e23531427a7df7541e6327 Mon Sep 17 00:00:00 2001 From: Varun Sridharan Date: Sun, 8 May 2022 07:00:54 +0530 Subject: [PATCH 2/4] Revert "Add a blacklist for repositories" --- README.md | 23 +++++++++++------------ src/config.json | 1 - src/helper.py | 12 ++---------- src/repositoryForked.py | 6 +----- src/repositorySource.py | 6 +----- src/repositoryStared.py | 6 +----- 6 files changed, 16 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index a666540..9b9b793 100644 --- a/README.md +++ b/README.md @@ -35,32 +35,31 @@ Or you can download the source code and install it where ever you need | --- | --- | | `github.username` | Your Github Username | | `github.accesstoken` | Your Github Account's Personal Access Token | -| - | - | +| - | - | | `gitea.host` | Selfhosted Gitea URL without `/` at the end | | `gitea.accesstoken` | Your Personal Access Token | | `gitea.username` | Account User Name | | `gitea.gist.prefix` | Custom Prefix For Repository When Mirroring Gists | | `gitea.gist.surfix` | Custom Prefix For Repository When Mirroring Gists | -| - | - | -| `repomap` | Remap A Repository To Diff User | -| `blacklist` | A list of glob-expression on github full repository names to skip | -| `gistsSource` | set to true to mirror all Gists Created By You| -| `gistsStared` | set to true to mirror all Gists Stared By You| -| `repositoryStared` | set to true to mirror all Repository Stared By You | -| `repositorySource` | set to true to mirror all Repository Created By You | -| `repositoryForked` | set to true to mirror all Repository Forked By You | +| - | - | +| `repomap` | Remap A Repository To Diff User | +| `gistsSource` | set to true to mirror all Gists Created By You| +| `gistsStared` | set to true to mirror all Gists Stared By You| +| `repositoryStared` | set to true to mirror all Repository Stared By You | +| `repositorySource` | set to true to mirror all Repository Created By You | +| `repositoryForked` | set to true to mirror all Repository Forked By You | | - | - | | `local_cache.enabled` | Set to **true** to store all repostiory slugs from gitea as json | | `local_cache.file_path` | Custom Path to store json file | -> Local Cache can come handly when running this script via cron +> Local Cache can come handly when running this script via cron > which reduces api request your selfhosted gitea instance ### Run cmd & Wait $ python3 mirror.py - - + + ## Cron Setup 1. Run `crontab -e` 2. `mkdir $HOME/mirrorLogs -p` diff --git a/src/config.json b/src/config.json index 426a03c..85335af 100644 --- a/src/config.json +++ b/src/config.json @@ -14,7 +14,6 @@ } }, "repomap" : { }, - "blacklist" : [ ], "gistsSource" : true, "gistsStared" : true, "repositoryStared" : true, diff --git a/src/helper.py b/src/helper.py index d110a95..d164131 100644 --- a/src/helper.py +++ b/src/helper.py @@ -5,7 +5,6 @@ import requests import json import sys import os -import fnmatch from datetime import datetime giteaGetUserCache = dict() @@ -79,9 +78,9 @@ def giteaCreateRepo(data,isPrivate,isRepository): data["service"] = 'github' data["wiki"] = True data["auth_token"] = "{0}".format(config['github']['accesstoken']) + - - + jsonstring = json.dumps(data) r = session.post(giteaHost('repos/migrate'), data=jsonstring) @@ -202,10 +201,3 @@ def giteaGetAllUsersOrgs(type): loopCount += 1 return results - - -def isBlacklistedRepository(full_name): - for pattern in config['blacklist']: - if fnmatch.fnmatch(full_name, pattern): - return True - return False diff --git a/src/repositoryForked.py b/src/repositoryForked.py index 0a4e47c..b803048 100644 --- a/src/repositoryForked.py +++ b/src/repositoryForked.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,10 +21,6 @@ def repositoryForked(): log('Forked Repository : {0}'.format(repo.full_name)) - if isBlacklistedRepository(repo.full_name): - print(" ---> Warning : Repository Matches Blacklist") - continue - if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositorySource.py b/src/repositorySource.py index 7f40a03..292c309 100644 --- a/src/repositorySource.py +++ b/src/repositorySource.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,10 +21,6 @@ def repositorySource(): log('Source Repository : {0}'.format(repo.full_name)) - if isBlacklistedRepository(repo.full_name): - print(" ---> Warning : Repository Matches Blacklist") - continue - if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositoryStared.py b/src/repositoryStared.py index ebe3513..ac72a44 100644 --- a/src/repositoryStared.py +++ b/src/repositoryStared.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import isBlacklistedRepository, log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config +from helper import log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -20,10 +20,6 @@ def repositoryStared(): log('⭐ Star\'ed Repository : {0}'.format(repo.full_name)) - if isBlacklistedRepository(repo.full_name): - print(" ---> Warning : Repository Matches Blacklist") - continue - if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] From d3c463782d08280c832e33ec2322ca0da8944e28 Mon Sep 17 00:00:00 2001 From: Markus Kaiser Date: Tue, 3 May 2022 17:09:49 +0200 Subject: [PATCH 3/4] Add a blacklist for repositories --- README.md | 23 ++++++++++++----------- src/config.json | 1 + src/helper.py | 12 ++++++++++-- src/repositoryForked.py | 6 +++++- src/repositorySource.py | 6 +++++- src/repositoryStared.py | 6 +++++- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9b9b793..a666540 100644 --- a/README.md +++ b/README.md @@ -35,31 +35,32 @@ Or you can download the source code and install it where ever you need | --- | --- | | `github.username` | Your Github Username | | `github.accesstoken` | Your Github Account's Personal Access Token | -| - | - | +| - | - | | `gitea.host` | Selfhosted Gitea URL without `/` at the end | | `gitea.accesstoken` | Your Personal Access Token | | `gitea.username` | Account User Name | | `gitea.gist.prefix` | Custom Prefix For Repository When Mirroring Gists | | `gitea.gist.surfix` | Custom Prefix For Repository When Mirroring Gists | -| - | - | -| `repomap` | Remap A Repository To Diff User | -| `gistsSource` | set to true to mirror all Gists Created By You| -| `gistsStared` | set to true to mirror all Gists Stared By You| -| `repositoryStared` | set to true to mirror all Repository Stared By You | -| `repositorySource` | set to true to mirror all Repository Created By You | -| `repositoryForked` | set to true to mirror all Repository Forked By You | +| - | - | +| `repomap` | Remap A Repository To Diff User | +| `blacklist` | A list of glob-expression on github full repository names to skip | +| `gistsSource` | set to true to mirror all Gists Created By You| +| `gistsStared` | set to true to mirror all Gists Stared By You| +| `repositoryStared` | set to true to mirror all Repository Stared By You | +| `repositorySource` | set to true to mirror all Repository Created By You | +| `repositoryForked` | set to true to mirror all Repository Forked By You | | - | - | | `local_cache.enabled` | Set to **true** to store all repostiory slugs from gitea as json | | `local_cache.file_path` | Custom Path to store json file | -> Local Cache can come handly when running this script via cron +> Local Cache can come handly when running this script via cron > which reduces api request your selfhosted gitea instance ### Run cmd & Wait $ python3 mirror.py - - + + ## Cron Setup 1. Run `crontab -e` 2. `mkdir $HOME/mirrorLogs -p` diff --git a/src/config.json b/src/config.json index 85335af..426a03c 100644 --- a/src/config.json +++ b/src/config.json @@ -14,6 +14,7 @@ } }, "repomap" : { }, + "blacklist" : [ ], "gistsSource" : true, "gistsStared" : true, "repositoryStared" : true, diff --git a/src/helper.py b/src/helper.py index d164131..d110a95 100644 --- a/src/helper.py +++ b/src/helper.py @@ -5,6 +5,7 @@ import requests import json import sys import os +import fnmatch from datetime import datetime giteaGetUserCache = dict() @@ -78,9 +79,9 @@ def giteaCreateRepo(data,isPrivate,isRepository): data["service"] = 'github' data["wiki"] = True data["auth_token"] = "{0}".format(config['github']['accesstoken']) - - + + jsonstring = json.dumps(data) r = session.post(giteaHost('repos/migrate'), data=jsonstring) @@ -201,3 +202,10 @@ def giteaGetAllUsersOrgs(type): loopCount += 1 return results + + +def isBlacklistedRepository(full_name): + for pattern in config['blacklist']: + if fnmatch.fnmatch(full_name, pattern): + return True + return False diff --git a/src/repositoryForked.py b/src/repositoryForked.py index b803048..0a4e47c 100644 --- a/src/repositoryForked.py +++ b/src/repositoryForked.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,6 +21,10 @@ def repositoryForked(): log('Forked Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositorySource.py b/src/repositorySource.py index 292c309..7f40a03 100644 --- a/src/repositorySource.py +++ b/src/repositorySource.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaCreateUserOrOrg,giteaSetRepoTopics,giteaSession,giteaCreateRepo,ghApi,giteaCreateOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -21,6 +21,10 @@ def repositorySource(): log('Source Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] diff --git a/src/repositoryStared.py b/src/repositoryStared.py index ac72a44..ebe3513 100644 --- a/src/repositoryStared.py +++ b/src/repositoryStared.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # https://github.com/PyGithub/PyGithub -from helper import log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config +from helper import isBlacklistedRepository, log,getConfig,giteaSetRepoTopics,giteaSession,giteaSetRepoStar,giteaCreateRepo,ghApi,giteaCreateUserOrOrg,giteaGetUser,config from github import GithubException from localCacheHelper import giteaExistsRepos,saveLocalCache import time @@ -20,6 +20,10 @@ def repositoryStared(): log('⭐ Star\'ed Repository : {0}'.format(repo.full_name)) + if isBlacklistedRepository(repo.full_name): + print(" ---> Warning : Repository Matches Blacklist") + continue + if real_repo in repo_map: gitea_dest_user = repo_map[real_repo] From 9f315504c075fb9049672e0e44fb60042c588517 Mon Sep 17 00:00:00 2001 From: Markus Kaiser Date: Tue, 17 May 2022 15:18:49 +0100 Subject: [PATCH 4/4] Ensure that an iterable blacklist exists --- src/helper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/helper.py b/src/helper.py index d110a95..333b23b 100644 --- a/src/helper.py +++ b/src/helper.py @@ -205,7 +205,11 @@ def giteaGetAllUsersOrgs(type): def isBlacklistedRepository(full_name): - for pattern in config['blacklist']: + blacklist = config.get('blacklist', []) + if isinstance(blacklist, str): + blacklist = [blacklist] + + for pattern in blacklist: if fnmatch.fnmatch(full_name, pattern): return True return False