Skip to content

Commit

Permalink
Add basic GitHub issue import functionality
Browse files Browse the repository at this point in the history 
Issue: #27
  • Loading branch information
@dspinellis
dspinellis committed Aug 28, 2018
1 parent bd89a1c commit dbdd9f5
Show file tree
Hide file tree
Showing 3 changed files with 229 additions and 22 deletions.
7 changes: 7 additions & 0 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,9 @@
addons:
apt:
packages:
curl
jq

language: bash

script: make test
23 changes: 21 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,21 +91,39 @@ You use _git issue_ with the following sub-commands.
### synchronize with remote repository
* `git issue push`: Update remote repository with local changes.
* `git issue pull`: Update local repository with remote changes.
### import GitHub issues from an existing GitHub repository
* `git issue import`: Import (or update) all issues from an existing GitHub
repository.
If the import involves more than a dozen of issues or if the repository
is private, set the environment variable `GI_CURL_ARGS` to an argument
that when passed to the *curl* program will supply GitHub the appropriate
API authentication.
For example, run the following command.
```
export GI_CURL_ARGS='-H "Authorization: token badf00ddead9bfee8f3c19afc3c97c6db55fcfde"'
```
You can create the authorization token at
[this URL](https://github.com/settings/tokens/new).
### help and debug
* `git issue help`: Display help information about git issue.
* `git issue log`: Output a log of changes made
* `git issue git`: Run the specified Git command on the issues repository.

Issues and comments are specified through the SHA hash associated with the
commit that opened them.
parent of the commit that opened them.

## Internals
All data are stored under `.issues`, which should be placed under `.gitignore`,
if it will coexist with another Git-based project.
The directory contains the following elements.
* A `.git` directory contains the Git data associated with the issues.
* A `config` file with configuration data.
* A `templates` directory with message templates.
* An `imports` directory contains details about imported issues.
* Files under `import/github/`*user*`/`*repo*`/`*number* contain the
*git-issue* SHA corresponding to an imported GitHub *number* issue.
* The file import/github/`*user*`/`*repo*`/checkpoint` contains the SHA
of the last imported or updated issue. This can be used for merging
future updates.
* An `issues` directory contains the individual issues.
* Each issue is stored in a directory named `issues/xx/xxxxxxx...`,
where the x's are the SHA of the issue's initial commit.
Expand All @@ -116,6 +134,7 @@ The directory contains the following elements.
* A `tags` file containing the issue's tags, one in each line.
* A `watchers` file containing the emails of persons to be notified when the issue changes (one per line).
* An `assignee` file containing the email for the person assigned to the issue.
* A `templates` directory with message templates.

## Contributing
Contributions are welcomed through pull requests.
Expand Down
221 changes: 201 additions & 20 deletions git-issue.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ cdissues()
}

# Output the path of an issue given its SHA
# The scheme used for storing the issues is a two level directory
# structure where the first level consists of the first two SHA
# letters
#
# issue_path_full <SHA>
issue_path_full()
{
Expand Down Expand Up @@ -94,26 +98,36 @@ trans_abort()
git reset $start_sha
git clean -qfd
git checkout -- .
rm -f gh-header gh-body
echo 'Operation aborted' 1>&2
exit 1
}

# Exit with an error if the specified prerequisite command
# cannot be executed
prerequisite_command()
{
if ! $1 -help 2>/dev/null 1>&2 ; then
cat <<EOF 1>&2
The $1 command is not availabe through the configured path.
Please install it and/or configure your PATH variable.
Command aborted.
EOF
exit 1
fi
}

# Commit an issue's changes
# commit <summary> <message>
# commit <summary> <message> [<date>]
commit()
{
commit_summary=$1
shift
commit_message=$1
shift
if [ "$1" ]; then
commit_date=$1
else
commit_date=$(date -R)
fi
git commit --allow-empty -q --date="$commit_date" -m "$commit_summary
commit_summary=$1
shift
commit_message=$1
shift
git commit --allow-empty -q -m "$commit_summary
$commit_message" || trans_abort
$commit_message" "$@" || trans_abort
}

# Allow the user to edit the specified file
Expand Down Expand Up @@ -209,7 +223,7 @@ EOF
EOF
cat >README.md <<\EOF
This is an distributed issue tracking repository based on Git.
Visit [gi](https://github.com/dspinellis/gi) for more information.
Visit [git-issue](https://github.com/dspinellis/git-issue) for more information.
EOF
git add config README.md templates/comment templates/description
commit 'gi: Initialize issues repository' 'gi init'
Expand Down Expand Up @@ -242,8 +256,7 @@ sub_new()
shift $(($OPTIND - 1));

trans_start
date=$(date -R)
commit 'gi: Add issue' 'gi new mark' "$date"
commit 'gi: Add issue' 'gi new mark'
sha=$(git rev-parse HEAD)
path=$(issue_path_full $sha)
mkdir -p $path || trans_abort
Expand All @@ -255,7 +268,7 @@ sub_new()
edit $path/description || trans_abort
fi
git add $path/description $path/tags || trans_abort
commit 'gi: Add issue description' "gi new description $sha" "$date"
commit 'gi: Add issue description' "gi new description $sha"
echo "Added issue $(short_sha $sha)"
}

Expand Down Expand Up @@ -298,20 +311,20 @@ Date: %aD' $isha

# Tags
if [ -s $path/tags ] ; then
printf '%s' 'Tags:'
printf 'Tags:'
fmt $path/tags | sed 's/^/ /'
fi

# Watchers
if [ -s $path/watchers ] ; then
printf '%s' 'Watchers:'
printf 'Watchers:'
fmt $path/watchers | sed 's/^/ /'
fi

# Assignee
if [ -r $path/assignee ] ; then
printf '%s' 'Assigned-to: '
cat $path/assignee
printf 'Assigned-to:'
sed 's/^/ /' $path/assignee
fi

# Description
Expand Down Expand Up @@ -523,6 +536,171 @@ sub_edit()
echo "Edited issue $(short_sha $isha)"
}

# import: import issues from GitHub {{{1
usage_import()
{
cat <<\USAGE_import_EOF
gi import usage: git issue import provider user repo
Example: git issue import github torvalds linux
USAGE_import_EOF
exit 2
}

# Get a page using the GitHub API; abort transaction on error
# Header is saved in the file gh-header; body in gh-body
gh_api_get()
{
local url

url="$1"
if ! curl $GI_CURL_ARGS -I -s "$url" >gh-header ; then
echo 'GitHub connection failed' 1>&2
trans_abort
fi

if grep -q '^Status: 404' gh-header ; then
echo "Invalid URL: $url" 1>&2
trans_abort
fi

if ! curl $GI_CURL_ARGS -s "$url" >gh-body ; then
echo 'GitHub connection failed' 1>&2
trans_abort
fi
}

# Import GitHub issues stored in the file gh-body as JSON data
# gh_import_issues user repo
gh_import_issues()
{
local user repo
local i j issue_number import_file sha path begin_sha

user="$1"
repo="$2"

prerequisite_command jq
prerequisite_command curl

begin_sha=$(git rev-parse HEAD)

# For each issue in the gh-body file
for i in $(seq 0 $(($(jq '. | length' gh-body) - 1)) ) ; do
issue_number=$(jq ".[$i].number" gh-body)

# See if issue already there
import_file="imports/github/$user/$repo/$issue_number"
if [ -r "$import_file" ] ; then
sha=$(cat "$import_file")
else
sha=$(git rev-parse HEAD)
fi

path=$(issue_path_full $sha)
mkdir -p $path || trans_abort

# Add issue import number to allow future updates
echo $sha >"$import_file"

# Create tags (in sorted order to avoid gratuitous updates)
{
jq -r ".[$i].state" gh-body
for j in $(seq 0 $(($(jq ".[$i].labels | length" gh-body) - 1)) ) ; do
jq -r ".[$i].labels[$j].name" gh-body
done
} |
LC_ALL=C sort >$path/tags || trans_abort

# Create assignees (in sorted order to avoid gratuitous updates)
for j in $(seq 0 $(($(jq ".[$i].assignees | length" gh-body) - 1)) ) ; do
jq -r ".[$i].assignees[$j].login" gh-body
done |
LC_ALL=C sort >$path/assignee || trans_abort

if [ -s $path/assignee ] ; then
git add $path/assignee || trans_abort
else
rm -f $path/assignee
fi

# Create description
{
jq -r ".[$i].title" gh-body
echo
jq -r ".[$i].body" gh-body
} >$path/description || trans_abort

git add $path/description $path/tags imports || trans_abort
if ! git diff --quiet HEAD ; then
local name
name=$(jq -r ".[$i].user.login" gh-body)
GIT_AUTHOR_DATE=$(jq -r ".[$i].updated_at" gh-body) \
commit "gi: Import issue #$issue_number from GitHub" \
"Issue URL: https://github.com/$user/$repo/issues/$issue_number" \
--author="$name <$name@users.noreply.github.com>"
echo "Imported/updated issue #$issue_number as $(short_sha $sha)"
fi
done

# Mark last import SHA, so we can use this for merging
if [ $begin_sha != $(git rev-parse HEAD) ] ; then
local checkpoint="imports/github/$user/$repo/checkpoint"
git rev-parse HEAD >"$checkpoint"
git add "$checkpoint"
commit "gi: Import issues from GitHub checkpoint" \
"Issues URL: https://github.com/$user/$repo/issues"
fi
}

# Return the next page API URL specified in gh-header
# Header examples (easy and tricky)
# Link: <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=3>; rel="next", <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=3>; rel="last", <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=1>; rel="first"
# Link: <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=1>; rel="prev", <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=3>; rel="next", <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=3>; rel="last", <https://api.github.com/repositories/146456308/issues?state=all&per_page=1&page=1>; rel="first"
gh_next_page_url()
{
sed -n '
:again
# Print "next" link
# This works only for the first element of the Link header
s/^Link:.<\([^>]*\)>; rel="next".*/\1/p
# If substitution worked branch to end of script
t
# Remove first element of the Link header and retry
s/^Link: <[^>]*>; rel="[^"]*", */Link: /
t again
' gh-header
}

# Import issues from specified source (currently github)
sub_import()
{
local endpoint user repo

test "$1" = github -a -n "$2" -a -n "$3" || usage_import
user="$2"
repo="$3"

cdissues


# Process GitHub issues page by page
trans_start
mkdir -p "imports/github/$user/$repo"
endpoint="https://api.github.com/repos/$user/$repo/issues?state=all"
while true ; do
gh_api_get "$endpoint"
gh_import_issues "$user" "$repo"

# Return if no more pages
if ! grep -q '^Link:.*rel="next"' gh-header ; then
break
fi

# Move to next point
endpoint=$(gh_next_page_url)
done
}

# list: Show issues matching a tag {{{1
usage_list()
{
Expand Down Expand Up @@ -675,6 +853,9 @@ case "$subcommand" in
clone) # Clone specified remote directory.
sub_clone "$@"
;;
import) # Import issues from specified source
sub_import "$@"
;;
new) # Create a new issue and mark it as open.
sub_new "$@"
;;
Expand Down

0 comments on commit dbdd9f5

Please sign in to comment.