Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into feature/ppca
Browse files Browse the repository at this point in the history
  • Loading branch information
ProfFan committed Nov 24, 2020
2 parents 7e018e8 + addb412 commit 4fdf769
Show file tree
Hide file tree
Showing 11 changed files with 516 additions and 36 deletions.
33 changes: 21 additions & 12 deletions Package.resolved
Original file line number Diff line number Diff line change
@@ -1,12 +1,21 @@
{
"object": {
"pins": [
{
"package": "CSV.swift",
"repositoryURL": "https://github.com/yaslab/CSV.swift.git",
"state": {
"branch": null,
"revision": "81d2874c51db364d7e1d71b0d99018a294c87ac1",
"version": "2.4.3"
}
},
{
"package": "Penguin",
"repositoryURL": "https://github.com/saeta/penguin.git",
"state": {
"branch": "master",
"revision": "bb3118efc29779b123caa7a83f1694d9b92e9fa7",
"revision": "88bc092f4611e6acfa734b2e888880dcb65c0be1",
"version": null
}
},
Expand All @@ -20,30 +29,30 @@
}
},
{
"package": "swift-argument-parser",
"repositoryURL": "https://github.com/apple/swift-argument-parser",
"package": "Benchmark",
"repositoryURL": "https://github.com/google/swift-benchmark.git",
"state": {
"branch": null,
"revision": "3e7d2fe99da091dcc1e4a7dd22fc3cfc2dca7937",
"version": "0.2.2"
"revision": "8e0ef8bb7482ab97dcd2cd1d6855bd38921c345d",
"version": "0.1.0"
}
},
{
"package": "Benchmark",
"repositoryURL": "https://github.com/google/swift-benchmark.git",
"package": "swift-models",
"repositoryURL": "https://github.com/tensorflow/swift-models.git",
"state": {
"branch": null,
"revision": "f70bf472b00aeaa05e2374373568c2fe459c11c7",
"revision": "b2fc0325bf9d476bf2d7a4cd0a09d36486c506e4",
"version": null
}
},
{
"package": "swift-models",
"repositoryURL": "https://github.com/tensorflow/swift-models.git",
"package": "SwiftProtobuf",
"repositoryURL": "https://github.com/apple/swift-protobuf.git",
"state": {
"branch": null,
"revision": "c67c9fc024d811e4134f379205ce49dd530f593a",
"version": null
"revision": "426bef0ddfdc6449b7bd1418baafad6bbc56110b",
"version": "1.13.0"
}
},
{
Expand Down
30 changes: 17 additions & 13 deletions Package.swift
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// swift-tools-version:5.1
// swift-tools-version:5.3
// The swift-tools-version declares the minimum version of Swift required to build this package.

import PackageDescription
Expand All @@ -24,22 +24,26 @@ let package = Package(
dependencies: [
// Dependencies declare other packages that this package depends on.
// .package(url: /* package url */, from: "1.0.0"),
.package(url: "https://github.com/google/swift-benchmark.git", .branch("f70bf472b00aeaa05e2374373568c2fe459c11c7")),
.package(name: "Benchmark", url: "https://github.com/google/swift-benchmark.git", from: "0.1.0"),

.package(url: "https://github.com/saeta/penguin.git", .branch("master")),
.package(name: "Penguin", url: "https://github.com/saeta/penguin.git", .branch("master")),

.package(url: "https://github.com/ProfFan/tensorboardx-s4tf.git", from: "0.1.3"),
.package(name: "TensorBoardX", url: "https://github.com/ProfFan/tensorboardx-s4tf.git", from: "0.1.3"),
.package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("swift-5.2-branch")),
.package(url: "https://github.com/tensorflow/swift-models.git", .branch("c67c9fc024d811e4134f379205ce49dd530f593a")),
.package(url: "https://github.com/apple/swift-argument-parser", from: "0.2.0"),
.package(url: "https://github.com/vojtamolda/Plotly.swift", from: "0.4.0"),
.package(url: "https://github.com/apple/swift-argument-parser.git", from: "0.3.0"),
.package(url: "https://github.com/tensorflow/swift-models.git", .branch("b2fc0325bf9d476bf2d7a4cd0a09d36486c506e4")),
.package(name: "Plotly", url: "https://github.com/vojtamolda/Plotly.swift", from: "0.4.0"),
],
targets: [
// Targets are the basic building blocks of a package. A target can define a module or a test suite.
// Targets can depend on other targets in this package, and on products in packages which this package depends on.
.target(
name: "SwiftFusion",
dependencies: ["PenguinStructures", "PenguinTesting", "PenguinParallelWithFoundation"]),
dependencies: [
.product(name: "PenguinStructures", package: "Penguin"),
.product(name: "PenguinTesting", package: "Penguin"),
.product(name: "PenguinParallelWithFoundation", package: "Penguin")
]),
.target(
name: "SwiftFusionBenchmarks",
dependencies: [
Expand All @@ -62,14 +66,14 @@ let package = Package(
]),
.target(
name: "Pose3SLAMG2O",
dependencies: ["SwiftFusion", "TensorBoardX", "SwiftToolsSupport"],
dependencies: ["SwiftFusion", "TensorBoardX", .product(name: "SwiftToolsSupport", package: "swift-tools-support-core")],
path: "Examples/Pose3SLAMG2O"),
.target(
name: "BeeTrackingTool",
dependencies: [
"BeeDataset",
"BeeTracking",
"PenguinParallelWithFoundation",
.product(name: "PenguinParallelWithFoundation", package: "Penguin"),
.product(name: "ArgumentParser", package: "swift-argument-parser"),
],
path: "Examples/BeeTrackingTool"),
Expand All @@ -78,7 +82,7 @@ let package = Package(
dependencies: [
"BeeDataset",
"BeeTracking",
"PenguinParallelWithFoundation",
.product(name: "PenguinParallelWithFoundation", package: "Penguin"),
"SwiftFusion",
"Plotly",
.product(name: "ArgumentParser", package: "swift-argument-parser"),
Expand All @@ -89,7 +93,7 @@ let package = Package(
dependencies: [
"BeeDataset",
"BeeTracking",
"PenguinParallelWithFoundation",
.product(name: "PenguinParallelWithFoundation", package: "Penguin"),
"SwiftFusion",
"Plotly",
.product(name: "ArgumentParser", package: "swift-argument-parser"),
Expand All @@ -99,7 +103,7 @@ let package = Package(
name: "SwiftFusionTests",
dependencies: [
"SwiftFusion",
"PenguinTesting",
.product(name: "PenguinTesting", package: "Penguin"),
.product(name: "ModelSupport", package: "swift-models"),
]),
.testTarget(
Expand Down
67 changes: 60 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,78 @@ Think factor graphs a la [GTSAM](https://gtsam.org/) coupled with deep learning,

Still very early, but feel free to explore! Subject to *massive* change :-)

## Run tests
## Getting Started

### Swift on MacOS

Using XCode is the easiest way to develop on Mac. Unfortunately, the current version of Xcode 12 has a bug, and you need to work with the XCode beta, available from Apple [here](https://developer.apple.com/download/).

Follow the instructions to [install Swift for TensorFlow on MacOS](https://github.com/tensorflow/swift/blob/master/Installation.md#macos). Installing the latest development snapshot is recommended.

To use experimental toolchain:
```
export PATH="/Library/Developer/Toolchains/swift-tensorflow-RELEASE-0.11.xctoolchain/usr/bin/:$PATH"
```
swift test

To re-generate XCode project:
```
swift package generate-xcodeproj
```

## Run benchmarks
### Installing Swift on Linux

Requirements: Ubuntu 18.04 (if you use GPU).

Follow the instructions to [install Swift for TensorFlow on Linux](https://github.com/tensorflow/swift/blob/master/Installation.md#linux). Installing the latest development snapshot is recommended.

### Installing S4TF on Jetson Devices

Download toolchain [here](https://storage.googleapis.com/swift-tensorflow-artifacts/oneoff-builds/swift-tensorflow-RELEASE-0.11-Jetson4.4.tar.gz), and then follow Linux instructions.

### Run tests

To check whether everything works you can run all the tests by changing to the SwiftFusion directory and
```
swift run -c release -Xswiftc -cross-module-optimization SwiftFusionBenchmarks
swift test --enable-test-discovery
```

## Update dependency versions
### Run benchmarks

```
swift package update
swift run -c release -Xswiftc -cross-module-optimization SwiftFusionBenchmarks
```

## Working with VS Code

To enable autocomplete in VSCode, install the plugin vknabel.vscode-swift-development-environment, and set the following plugin settings:

- "sde.languageServerMode": "sourcekit-lsp",
- "sourcekit-lsp.serverPath": "<your toolchain path>/usr/bin/sourcekit-lsp",
- "sourcekit-lsp.toolchainPath": "<your toolchain path>",
- "swift.path.swift_driver_bin": "<your toolchain path>/usr/bin/swift",

Debugging within VS code is easiest via the CodeLLDB plugin so you can debug in vscode. You need to set the following setting:
"lldb.library": "/swift-tensorflow-toolchain/usr/lib/liblldb.so"

A sample launch.json file:

```json
{
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug",
"program": "${workspaceFolder}/.build/x86_64-unknown-linux-gnu/debug/SwiftFusionPackageTests.xctest",
"args": ["--enable-test-discovery"],
"cwd": "${workspaceFolder}"
}
]
}
```

## Overview
## Code Overview

The main code is in Sources/SwiftFusion, which as a number of sub-directories:

Expand Down
3 changes: 2 additions & 1 deletion Sources/BeeDataset/BeeFrames.swift
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public struct BeeFrames: RandomAccessCollection {
public func index(after i: Int) -> Int { i + 1 }

public subscript(index: Int) -> Tensor<Double> {
return Tensor<Double>(Image(contentsOf: directory.appendingPathComponent("frame\(index + 1).png")).tensor)
return Tensor<Double>(
Image(contentsOf: directory.appendingPathComponent("frame\(index + 1).png")).tensor)
}
}
2 changes: 1 addition & 1 deletion Sources/BeeTracking/AppearanceRAE.swift
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public struct DenseRAE: Layer {
}

/// Differentiable encoder
@differentiable
@differentiable(wrt: imageBatch)
public func encode(_ imageBatch: Tensor<Double>) -> Tensor<Double> {
let batchSize = imageBatch.shape[0]
let expectedShape: TensorShape = [batchSize, imageHeight, imageWidth, imageChannels]
Expand Down
116 changes: 116 additions & 0 deletions Sources/SwiftFusion/Geometry/Cal3_S2.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@

/// The five-parameter camera calibration.
public struct Cal3_S2: Manifold, Equatable {
public typealias Coordinate = Cal3_S2Coordinate
public typealias TangentVector = Vector5

public var coordinateStorage: Cal3_S2Coordinate

public init() {
self.init(coordinateStorage: Cal3_S2Coordinate())
}

public init(coordinateStorage: Cal3_S2Coordinate) {
self.coordinateStorage = coordinateStorage
}

/// Initializes from individual values.
public init(fx: Double, fy: Double, s: Double, u0: Double, v0: Double) {
self.init(coordinateStorage: Cal3_S2Coordinate(fx: fx, fy: fy, s: s, u0: u0, v0: v0))
}

/// Moves the parameter values by the specified direction.
public mutating func move(along direction: Vector5) {
coordinateStorage = coordinateStorage.retract(direction)
}
}

/// CameraCalibration conformance.
extension Cal3_S2: CameraCalibration {
@differentiable
public func uncalibrate(_ np: Vector2) -> Vector2 {
coordinate.uncalibrate(np)
}

@differentiable
public func calibrate(_ ip: Vector2) -> Vector2 {
coordinate.calibrate(ip)
}
}


/// Manifold coordinate for Cal3_S2.
public struct Cal3_S2Coordinate: Equatable {
/// Focal length in X direction.
public var fx: Double

/// Focal length in Y direction.
public var fy: Double

/// Skew factor.
public var s: Double

/// Image center in X direction.
public var u0: Double

/// Image center in Y direction.
public var v0: Double

/// Initializes from individual values.
public init(fx: Double, fy: Double, s: Double, u0: Double, v0: Double) {
self.fx = fx
self.fy = fy
self.s = s
self.u0 = u0
self.v0 = v0
}

/// Initializes from a vector.
public init(_ params: Vector5) {
self.fx = params.s0
self.fy = params.s1
self.s = params.s2
self.u0 = params.s3
self.v0 = params.s4
}

/// Initializes with default values, corresponding to the identity element.
public init() {
self.init(fx: 1.0, fy: 1.0, s: 0.0, u0: 0.0, v0: 0.0)
}

/// Returns the parameters as a vector.
public func asVector() -> Vector5 {
Vector5(fx, fy, s, u0, v0)
}
}

/// ManifoldCoordinate conformance.
extension Cal3_S2Coordinate: ManifoldCoordinate {
public typealias LocalCoordinate = Vector5

@differentiable(wrt: local)
public func retract(_ local: Vector5) -> Cal3_S2Coordinate {
Cal3_S2Coordinate(asVector() + local)
}

@differentiable(wrt: global)
public func localCoordinate(_ global: Cal3_S2Coordinate) -> Vector5 {
global.asVector() - asVector()
}
}

/// Operations on a point.
extension Cal3_S2Coordinate {
@differentiable
public func uncalibrate(_ np: Vector2) -> Vector2 {
Vector2(fx * np.x + s * np.y + u0, fy * np.y + v0)
}

@differentiable
public func calibrate(_ ip: Vector2) -> Vector2 {
let (du, dv) = (ip.x - u0, ip.y - v0)
let (fxInv, fyInv) = (1.0 / fx, 1.0 / fy)
return Vector2(fxInv * du - s * fxInv * fyInv * dv, fyInv * dv)
}
}
14 changes: 14 additions & 0 deletions Sources/SwiftFusion/Geometry/CameraCalibration.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@

/// A protocol for camera calibration parameters.
public protocol CameraCalibration: Differentiable {
/// Initializes to default (usually identity).
init()

/// Converts from image coordinate to normalized coordinate.
@differentiable
func calibrate(_ ip: Vector2) -> Vector2

/// Converts from normalized coordinate to image coordinate.
@differentiable
func uncalibrate(_ np: Vector2) -> Vector2
}
Loading

0 comments on commit 4fdf769

Please sign in to comment.