Skip to content
This repository was archived by the owner on Sep 13, 2024. It is now read-only.

Commit 613b26b

Browse files
peterjc123peterjc123
authored
Fix bugs around git clone and use yaml-based CI (#42)
Fix git clone Add azure pipeline yamls Use squash for auto merge
1 parent c02159c commit 613b26b

File tree

10 files changed

+262
-1
lines changed

10 files changed

+262
-1
lines changed

‎.github/auto-merge.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ maxRequestedChanges:
1010
NONE: 0
1111
blockingLabels:
1212
- blocked
13+
mergeMethod: squash
1314

1415
# Will merge whenever the above conditions are met, but also
1516
# the owner has approved or merge label was added.

‎.github/azure-pipelines-scripts/7z_install.bat

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
@echo off
2+
3+
curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
4+
if errorlevel 1 exit /b 1
5+
6+
start /wait 7z1805-x64.exe /S
7+
if errorlevel 1 exit /b 1
8+
9+
set "PATH=%ProgramFiles%\7-Zip;%PATH%"

‎.github/azure-pipelines-scripts/conda_install.bat

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
@echo off
2+
3+
:: Install Miniconda3
4+
set "CONDA_HOME=%CD%\conda"
5+
set "tmp_conda=%CONDA_HOME%"
6+
set "miniconda_exe=%CD%\miniconda.exe"
7+
if exist conda rmdir /s /q conda
8+
if exist miniconda.exe del miniconda.exe
9+
10+
curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
11+
if errorlevel 1 exit /b 1
12+
13+
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%
14+
if errorlevel 1 exit /b 1
15+
16+
set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
17+
18+
conda install numpy pyyaml mkl
19+
pip install ninja

‎.github/azure-pipelines-scripts/env_fix.bat

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
@echo off
2+
3+
:: Caution: Please don't use this script locally
4+
:: It may destroy your build environment.
5+
6+
setlocal
7+
8+
IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" (
9+
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
10+
exit /b 1
11+
)
12+
13+
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
14+
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
15+
set "VS15INSTALLDIR=%%i"
16+
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
17+
goto vswhere
18+
)
19+
)
20+
21+
:vswhere
22+
23+
IF "%VS15VCVARSALL%"=="" (
24+
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
25+
exit /b 1
26+
)
27+
28+
call "%VS15VCVARSALL%" x86_amd64
29+
for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak"
30+
31+
endlocal

‎.github/azure-pipelines-scripts/mkl_install.bat

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
@echo off
2+
3+
::MKL
4+
curl -k https://s3.amazonaws.com/ossci-windows/mkl_2018.2.185.7z --output mkl.7z
5+
if errorlevel 1 exit /b 1
6+
7+
7z x -aoa mkl.7z -omkl
8+
if errorlevel 1 exit /b 1
9+
10+
set CMAKE_INCLUDE_PATH=%cd%\mkl\include
11+
set LIB=%cd%\mkl\lib;%LIB%

‎.github/azure-pipelines-scripts/sccache_debug.bat

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
@echo off
2+
3+
IF NOT "%USE_SCCACHE%" == "false" (
4+
sccache -s
5+
type %SCCACHE_ERROR_LOG%
6+
)

‎.github/azure-pipelines-scripts/sccache_setup.bat

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
@echo off
2+
3+
:: SCCACHE
4+
IF NOT "%USE_SCCACHE%" == "false" (
5+
git clone https://ccache:PushFiles123@peter-jc.visualstudio.com/DefaultCollection/peter_jiachen/_git/peter_jiachen > nul 2>&1
6+
IF ERRORLEVEL 1 echo Clone cache failed
7+
8+
set SCCACHE_DIR=%CD%\peter_jiachen
9+
set SCCACHE_CACHE_SIZE=1
10+
set SCCACHE_NO_DAEMON=1
11+
set SCCACHE_ERROR_LOG=%CD%\error.log
12+
set RUST_LOG=sccache::compiler=debug
13+
14+
mkdir %CD%\tmp_bin
15+
16+
curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\tmp_bin\sccache.exe
17+
18+
set "PATH=%CD%\tmp_bin;%PATH%"
19+
20+
sccache --stop-server
21+
sccache --start-server
22+
sccache --zero-stats
23+
)

‎.github/azure-pipelines-scripts/sccache_update.bat

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
@echo off
2+
3+
IF NOT "%USE_SCCACHE%" == "false" (
4+
cd peter_jiachen
5+
attrib -s -h -r . /s /d
6+
rmdir /s /q .git
7+
8+
git config --global user.name "Azure DevOps"
9+
git config --global user.email peter_jiachen@163.com
10+
git init
11+
git remote add origin %SCCACHE_REPO_URL%
12+
git add .
13+
git commit -m "Update cache"
14+
git push origin master -f
15+
)

‎.github/azure-pipelines.yml

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
2+
variables:
3+
pytorch.branch: master
4+
pytorch.repo: pytorch
5+
build.pytorch.enabled: true
6+
build.caffe.enabled: true
7+
use.sccache: false
8+
9+
jobs:
10+
- job: 'PyTorch_Test'
11+
timeoutInMinutes: 0
12+
cancelTimeoutInMinutes: 5
13+
condition: eq(variables['build.pytorch.enabled'], 'true')
14+
15+
pool:
16+
vmImage: 'vs2017-win2016'
17+
18+
steps:
19+
- task: BatchScript@1
20+
displayName: 'Environment fix'
21+
inputs:
22+
filename: '.github/azure-pipelines-scripts/env_fix.bat'
23+
24+
modifyEnvironment: false
25+
26+
- task: BatchScript@1
27+
displayName: 'Install 7Zip'
28+
inputs:
29+
filename: '.github/azure-pipelines-scripts/7z_install.bat'
30+
31+
modifyEnvironment: true
32+
33+
- script: 'choco install -y 7Zip' # Not stable
34+
displayName: 'Install 7Zip'
35+
enabled: false
36+
37+
- task: BatchScript@1
38+
displayName: 'Install Conda'
39+
inputs:
40+
filename: '.github/azure-pipelines-scripts/conda_install.bat'
41+
42+
modifyEnvironment: true
43+
44+
- task: BatchScript@1
45+
displayName: 'Install MKL'
46+
inputs:
47+
filename: '.github/azure-pipelines-scripts/mkl_install.bat'
48+
49+
modifyEnvironment: true
50+
51+
- task: BatchScript@1
52+
displayName: 'Setup sccache'
53+
condition: eq(variables['use.sccache'], 'true')
54+
inputs:
55+
filename: '.github/azure-pipelines-scripts/sccache_setup.bat'
56+
57+
modifyEnvironment: true
58+
59+
- script: 'call cpu.bat /y'
60+
displayName: Build
61+
env:
62+
MAX_JOBS: 2
63+
SKIP_VS_VER_CHECK: 1
64+
65+
- task: BatchScript@1
66+
displayName: 'Debug sccache'
67+
condition: eq(variables['use.sccache'], 'true')
68+
inputs:
69+
filename: '.github/azure-pipelines-scripts/sccache_debug.bat'
70+
71+
modifyEnvironment: false
72+
73+
- task: BatchScript@1
74+
displayName: 'Update sccache'
75+
condition: eq(variables['use.sccache'], 'true')
76+
inputs:
77+
filename: '.github/azure-pipelines-scripts/sccache_update.bat'
78+
79+
modifyEnvironment: false
80+
81+
- job: 'Caffe2_Test'
82+
timeoutInMinutes: 0
83+
cancelTimeoutInMinutes: 5
84+
condition: eq(variables['build.caffe.enabled'], 'true')
85+
86+
pool:
87+
vmImage: 'vs2015-win2012r2'
88+
89+
steps:
90+
- task: BatchScript@1
91+
displayName: 'Install 7Zip'
92+
inputs:
93+
filename: '.github/azure-pipelines-scripts/7z_install.bat'
94+
95+
modifyEnvironment: true
96+
97+
- script: 'choco install -y 7Zip' # Not stable
98+
displayName: 'Install 7Zip'
99+
enabled: false
100+
101+
- task: BatchScript@1
102+
displayName: 'Install Conda'
103+
inputs:
104+
filename: '.github/azure-pipelines-scripts/conda_install.bat'
105+
106+
modifyEnvironment: true
107+
108+
- task: BatchScript@1
109+
displayName: 'Install MKL'
110+
inputs:
111+
filename: '.github/azure-pipelines-scripts/mkl_install.bat'
112+
113+
modifyEnvironment: true
114+
115+
- task: BatchScript@1
116+
displayName: 'Setup sccache'
117+
condition: eq(variables['use.sccache'], 'true')
118+
inputs:
119+
filename: '.github/azure-pipelines-scripts/sccache_setup.bat'
120+
121+
modifyEnvironment: true
122+
123+
- script: 'if exist caffe2_builders/v140/auto.bat call caffe2_builders/v140/auto.bat'
124+
displayName: Build
125+
env:
126+
MAX_JOBS: 2
127+
BUILD_RELEASE: 1
128+
BUILD_DEBUG: 0
129+
130+
- task: BatchScript@1
131+
displayName: 'Debug sccache'
132+
condition: eq(variables['use.sccache'], 'true')
133+
inputs:
134+
filename: '.github/azure-pipelines-scripts/sccache_debug.bat'
135+
136+
modifyEnvironment: false
137+
138+
- task: BatchScript@1
139+
displayName: 'Update sccache'
140+
condition: eq(variables['use.sccache'], 'true')
141+
inputs:
142+
filename: '.github/azure-pipelines-scripts/sccache_update.bat'
143+
144+
modifyEnvironment: false

‎internal/clone.bat

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
IF "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
44

5-
git clone --recursive --jobs %NUMBER_OF_PROCESSORS% https://github.com/%PYTORCH_REPO%/pytorch
5+
git clone https://github.com/%PYTORCH_REPO%/pytorch
66

77
cd pytorch
88

@@ -22,3 +22,5 @@ IF NOT "%PYTORCH_BUILD_VERSION%"=="" (
2222
echo Building for branch %PYTORCH_BRANCH%
2323
)
2424
)
25+
26+
git submodule update --init --recursive

0 commit comments

Comments
 (0)