runtime,cmd/internal/obj/arm64,cmd/internal/objabi,cmd/link/internal/arm64: enable conditional branches on target outside of asm file

Results of running runtime hash benchmarks:
goos: linux
goarch: arm64
pkg: runtime
                       │ master_noaes_runtime_hash.txt │ arm64-condbr19_noaes_runtime_hash.txt │
                       │            sec/op             │     sec/op      vs base               │
Hash5-4                                    9.628n ± 0%      8.858n ± 0%  -8.00% (p=0.000 n=25)
Hash16-4                                   9.627n ± 0%      9.627n ± 0%       ~ (p=0.117 n=25)
Hash64-4                                   15.03n ± 0%      14.25n ± 0%  -5.19% (p=0.000 n=25)
Hash1024-4                                 90.19n ± 0%      88.19n ± 0%  -2.22% (p=0.000 n=25)
Hash65536-4                                5.058µ ± 0%      5.192µ ± 0%  +2.65% (p=0.000 n=25)
HashStringSpeed-4                          19.00n ± 3%      18.46n ± 4%       ~ (p=0.096 n=25)
HashBytesSpeed-4                           65.26n ± 0%      64.36n ± 0%  -1.38% (p=0.000 n=25)
HashInt32Speed-4                           12.46n ± 0%      12.50n ± 1%       ~ (p=0.829 n=25)
HashInt64Speed-4                           13.01n ± 3%      12.47n ± 3%  -4.15% (p=0.000 n=25)
HashStringArraySpeed-4                     43.45n ± 2%      41.55n ± 3%  -4.37% (p=0.000 n=25)
FastrandHashiter-4                         11.72n ± 1%      11.81n ± 1%       ~ (p=0.079 n=25)
geomean                                    33.77n           33.00n       -2.26%

            │ master_noaes_runtime_hash.txt │ arm64-condbr19_noaes_runtime_hash.txt │
            │              B/s              │      B/s        vs base               │
Hash5-4                        495.3Mi ± 0%     538.3Mi ± 0%  +8.69% (p=0.000 n=25)
Hash16-4                       1.548Gi ± 0%     1.548Gi ± 0%  +0.00% (p=0.025 n=25)
Hash64-4                       3.966Gi ± 0%     4.183Gi ± 0%  +5.47% (p=0.000 n=25)
Hash1024-4                     10.57Gi ± 0%     10.81Gi ± 0%  +2.27% (p=0.000 n=25)
Hash65536-4                    12.07Gi ± 0%     11.76Gi ± 0%  -2.57% (p=0.000 n=25)
geomean                        3.279Gi          3.367Gi       +2.70%

These results are obtained with cpu.ARM64.HasAES == false

Change-Id: If282267b9b2dcf474516cb33bfdbda4ee35bb8fa

Author: ArsenySamoylov

src/cmd/internal/obj/arm64/asm7.go

@@ -2691,6 +2691,11 @@ func cmp(a int, b int) bool {
 			return true
 		}
 
+	case C_SBRA:
+		if b == C_ADDR {
+			return true
+		}
+
 	case C_LBRA:
 		if b == C_SBRA {
 			return true
@@ -4260,7 +4265,18 @@ func (c *ctxt7) asmout(p *obj.Prog, o *Optab, out []uint32) {
 		o1 = c.opirr(p, p.As)
 
 		o1 |= uint32(p.From.Reg & 31)
-		o1 |= uint32(c.brdist(p, 0, 19, 2) << 5)
+
+		if p.To.Sym == nil {
+			o1 |= uint32(c.brdist(p, 0, 19, 2) << 5)
+			break
+		}
+
+		rel := obj.Addrel(c.cursym)
+		rel.Off = int32(c.pc)
+		rel.Siz = 4
+		rel.Sym = p.To.Sym
+		rel.Add = p.To.Offset
+		rel.Type = objabi.R_AARCH64_CONDBR19
 
 	case 40: /* tbz */
 		o1 = c.opirr(p, p.As)

src/cmd/internal/obj/arm64/asm_arm64_test.go

@@ -38,3 +38,17 @@ func TestMOVK(t *testing.T) {
 		t.Errorf("Got %x want %x\n", x, want)
 	}
 }
+
+func testcondbr19() uint64
+func testcondbr19_target() uint64 {
+	return 42
+}
+
+// TestCondBr19 makes sure that conditional branch on labels outside of *arm64.s file works fine
+func TestCondBr19(t *testing.T) {
+	x := testcondbr19()
+	want := uint64(42)
+	if x != want {
+		t.Errorf("Got %s want %d\n", x, want)
+	}
+}

src/cmd/internal/obj/arm64/asm_arm64_test.s

@@ -37,3 +37,8 @@ TEXT ·testmovk(SB), NOSPLIT, $0-8
 	MOVK	$(40000<<48), R0
 	MOVD	R0, ret+0(FP)
 	RET
+
+// testcondbr19() uint64
+TEXT ·testcondbr19(SB), NOSPLIT, $0-8
+	MOVD	$0, R0
+	CBZ R0, ·testcondbr19_target(SB)

src/cmd/internal/objabi/reloctype.go

@@ -131,6 +131,7 @@ const (
 	// inherently processor specific.
 
 	// Arm64.
+	R_AARCH64_CONDBR19
 
 	// Set a MOV[NZ] immediate field to bits [15:0] of the offset from the thread
 	// local base to the thread local variable defined by the referenced (thread

src/cmd/internal/objabi/reloctype_string.go

@@ -968,6 +968,18 @@ func archreloc(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, r loade
 		}
 		return val | ((t >> 2) & 0x03ffffff), noExtReloc, true
 
+	case objabi.R_AARCH64_CONDBR19:
+		var t int64
+		if ldr.SymType(rs) == sym.SDYNIMPORT {
+			t = (ldr.SymAddr(syms.PLT) + r.Add()) - (ldr.SymValue(s) + int64(r.Off()))
+		} else {
+			t = (ldr.SymAddr(rs) + r.Add()) - (ldr.SymValue(s) + int64(r.Off()))
+		}
+		if t >= 1<<20 || t < -1<<20 {
+			ldr.Errorf(s, "program too large, call relocation distance = %d", t)
+		}
+		return val | (((t >> 2) & 0x7ffff) << 5), noExtReloc, true
+
 	case objabi.R_ARM64_GOT:
 		if (val>>24)&0x9f == 0x90 {
 			// R_AARCH64_ADR_GOT_PAGE

src/cmd/link/internal/arm64/asm.go

@@ -532,7 +532,7 @@ CALLFN(·call1073741824, 1073741824)
 // func memhash32(p unsafe.Pointer, h uintptr) uintptr
 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
 	MOVB	runtime·useAeshash(SB), R10
-	CBZ	R10, noaes
+	CBZ	R10, runtime·memhash32Fallback<ABIInternal>(SB)
 	MOVD	$runtime·aeskeysched+0(SB), R3
 
 	VEOR	V0.B16, V0.B16, V0.B16
@@ -548,13 +548,11 @@ TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
 
 	VMOV	V0.D[0], R0
 	RET
-noaes:
-	B	runtime·memhash32Fallback<ABIInternal>(SB)
 
 // func memhash64(p unsafe.Pointer, h uintptr) uintptr
 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
 	MOVB	runtime·useAeshash(SB), R10
-	CBZ	R10, noaes
+	CBZ	R10, runtime·memhash64Fallback<ABIInternal>(SB)
 	MOVD	$runtime·aeskeysched+0(SB), R3
 
 	VEOR	V0.B16, V0.B16, V0.B16
@@ -570,25 +568,19 @@ TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
 
 	VMOV	V0.D[0], R0
 	RET
-noaes:
-	B	runtime·memhash64Fallback<ABIInternal>(SB)
 
 // func memhash(p unsafe.Pointer, h, size uintptr) uintptr
 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
 	MOVB	runtime·useAeshash(SB), R10
-	CBZ	R10, noaes
+	CBZ	R10, runtime·memhashFallback<ABIInternal>(SB)
 	B	aeshashbody<>(SB)
-noaes:
-	B	runtime·memhashFallback<ABIInternal>(SB)
 
 // func strhash(p unsafe.Pointer, h uintptr) uintptr
 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
 	MOVB	runtime·useAeshash(SB), R10
-	CBZ	R10, noaes
+	CBZ	R10, runtime·strhashFallback<ABIInternal>(SB)
 	LDP	(R0), (R0, R2)	// string data / length
 	B	aeshashbody<>(SB)
-noaes:
-	B	runtime·strhashFallback<ABIInternal>(SB)
 
 // R0: data
 // R1: seed data