aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iio/dac/ad5446.c
blob: 7df8b4cc295ddeedbcc88372a6d54aabc915b5d4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * AD5446 SPI DAC driver
 *
 * Copyright 2010 Analog Devices Inc.
 */

#include <linux/interrupt.h>
#include <linux/workqueue.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/sysfs.h>
#include <linux/list.h>
#include <linux/spi/spi.h>
#include <linux/i2c.h>
#include <linux/regulator/consumer.h>
#include <linux/err.h>
#include <linux/module.h>

#include <linux/iio/iio.h>
#include <linux/iio/sysfs.h>

#define MODE_PWRDWN_1k		0x1
#define MODE_PWRDWN_100k	0x2
#define MODE_PWRDWN_TRISTATE	0x3

/**
 * struct ad5446_state - driver instance specific data
 * @spi:		spi_device
 * @chip_info:		chip model specific constants, available modes etc
 * @reg:		supply regulator
 * @vref_mv:		actual reference voltage used
 */

struct ad5446_state {
	struct device		*dev;
	const struct ad5446_chip_info	*chip_info;
	struct regulator		*reg;
	unsigned short			vref_mv;
	unsigned			cached_val;
	unsigned			pwr_down_mode;
	unsigned			pwr_down;
};

/**
 * struct ad5446_chip_info - chip specific information
 * @channel:		channel spec for the DAC
 * @int_vref_mv:	AD5620/40/60: the internal reference voltage
 * @write:		chip specific helper function to write to the register
 */

struct ad5446_chip_info {
	struct iio_chan_spec	channel;
	u16			int_vref_mv;
	int			(*write)(struct ad5446_state *st, unsigned val);
};

static const char * const ad5446_powerdown_modes[] = {
	"1kohm_to_gnd", "100kohm_to_gnd", "three_state"
};

static int ad5446_set_powerdown_mode(struct iio_dev *indio_dev,
	const struct iio_chan_spec *chan, unsigned int mode)
{
	struct ad5446_state *st = iio_priv(indio_dev);

	st->pwr_down_mode = mode + 1;

	return 0;
}

static int ad5446_get_powerdown_mode(struct iio_dev *indio_dev,
	const struct iio_chan_spec *chan)
{
	struct ad5446_state *st = iio_priv(indio_dev);

	return st->pwr_down_mode - 1;
}

static const struct iio_enum ad5446_powerdown_mode_enum = {
	.items = ad5446_powerdown_modes,
	.num_items = ARRAY_SIZE(ad5446_powerdown_modes),
	.get = ad5446_get_powerdown_mode,
	.set = ad5446_set_powerdown_mode,
};

static ssize_t ad5446_read_dac_powerdown(struct iio_dev *indio_dev,
					   uintptr_t private,
					   const struct iio_chan_spec *chan,
					   char *buf)
{
	struct ad5446_state *st = iio_priv(indio_dev);

	return sprintf(buf, "%d\n", st->pwr_down);
}

static ssize_t ad5446_write_dac_powerdown(struct iio_dev *indio_dev,
					    uintptr_t private,
					    const struct iio_chan_spec *chan,
					    const char *buf, size_t len)
{
	struct ad5446_state *st = iio_priv(indio_dev);
	unsigned int shift;
	unsigned int val;
	bool powerdown;
	int ret;

	ret = strtobool(buf, &powerdown);
	if (ret)
		return ret;

	mutex_lock(&indio_dev->mlock);
	st->pwr_down = powerdown;

	if (st->pwr_down) {
		shift = chan->scan_type.realbits + chan->scan_type.shift;
		val = st->pwr_down_mode << shift;
	} else {
		val = st->cached_val;
	}

	ret = st->chip_info->write(st, val);
	mutex_unlock(&indio_dev->mlock);

	return ret ? ret : len;
}

static const struct iio_chan_spec_ext_info ad5446_ext_info_powerdown[] = {
	{
		.name = "powerdown",
		.read = ad5446_read_dac_powerdown,
		.write = ad5446_write_dac_powerdown,
		.shared = IIO_SEPARATE,
	},
	IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad5446_powerdown_mode_enum),
	IIO_ENUM_AVAILABLE("powerdown_mode", &ad5446_powerdown_mode_enum),
	{ },
};

#define _AD5446_CHANNEL(bits, storage, _shift, ext) { \
	.type = IIO_VOLTAGE, \
	.indexed = 1, \
	.output = 1, \
	.channel = 0, \
	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
	.scan_type = { \
		.sign = 'u', \
		.realbits = (bits), \
		.storagebits = (storage), \
		.shift = (_shift), \
		}, \
	.ext_info = (ext), \
}

#define AD5446_CHANNEL(bits, storage, shift) \
	_AD5446_CHANNEL(bits, storage, shift, NULL)

#define AD5446_CHANNEL_POWERDOWN(bits, storage, shift) \
	_AD5446_CHANNEL(bits, storage, shift, ad5446_ext_info_powerdown)

static int ad5446_read_raw(struct iio_dev *indio_dev,
			   struct iio_chan_spec const *chan,
			   int *val,
			   int *val2,
			   long m)
{
	struct ad5446_state *st = iio_priv(indio_dev);

	switch (m) {
	case IIO_CHAN_INFO_RAW:
		*val = st->cached_val;
		return IIO_VAL_INT;
	case IIO_CHAN_INFO_SCALE:
		*val = st->vref_mv;
		*val2 = chan->scan_type.realbits;
		return IIO_VAL_FRACTIONAL_LOG2;
	}
	return -EINVAL;
}

static int ad5446_write_raw(struct iio_dev *indio_dev,
			       struct iio_chan_spec const *chan,
			       int val,
			       int val2,
			       long mask)
{
	struct ad5446_state *st = iio_priv(indio_dev);
	int ret = 0;

	switch (mask) {
	case IIO_CHAN_INFO_RAW:
		if (val >= (1 << chan->scan_type.realbits) || val < 0)
			return -EINVAL;

		val <<= chan->scan_type.shift;
		mutex_lock(&indio_dev->mlock);
		st->cached_val = val;
		if (!st->pwr_down)
			ret = st->chip_info->write(st, val);
		mutex_unlock(&indio_dev->mlock);
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}

static const struct iio_info ad5446_info = {
	.read_raw = ad5446_read_raw,
	.write_raw = ad5446_write_raw,
};

static int ad5446_probe(struct device *dev, const char *name,
			const struct ad5446_chip_info *chip_info)
{
	struct ad5446_state *st;
	struct iio_dev *indio_dev;
	struct regulator *reg;
	int ret, voltage_uv = 0;

	reg = devm_regulator_get(dev, "vcc");
	if (!IS_ERR(reg)) {
		ret = regulator_enable(reg);
		if (ret)
			return ret;

		ret = regulator_get_voltage(reg);
		if (ret < 0)
			goto error_disable_reg;

		voltage_uv = ret;
	}

	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
	if (indio_dev == NULL) {
		ret = -ENOMEM;
		goto error_disable_reg;
	}
	st = iio_priv(indio_dev);
	st->chip_info = chip_info;

	dev_set_drvdata(dev, indio_dev);
	st->reg = reg;
	st->dev = dev;

	/* Establish that the iio_dev is a child of the device */
	indio_dev->dev.parent = dev;
	indio_dev->name = name;
	indio_dev->info = &ad5446_info;
	indio_dev->modes = INDIO_DIRECT_MODE;
	indio_dev->channels = &st->chip_info->channel;
	indio_dev->num_channels = 1;

	st->pwr_down_mode = MODE_PWRDWN_1k;

	if (st->chip_info->int_vref_mv)
		st->vref_mv = st->chip_info->int_vref_mv;
	else if (voltage_uv)
		st->vref_mv = voltage_uv / 1000;
	else
		dev_warn(dev, "reference voltage unspecified\n");

	ret = iio_device_register(indio_dev);
	if (ret)
		goto error_disable_reg;

	return 0;

error_disable_reg:
	if (!IS_ERR(reg))
		regulator_disable(reg);
	return ret;
}

static int ad5446_remove(struct device *dev)
{
	struct iio_dev *indio_dev = dev_get_drvdata(dev);
	struct ad5446_state *st = iio_priv(indio_dev);

	iio_device_unregister(indio_dev);
	if (!IS_ERR(st->reg))
		regulator_disable(st->reg);

	return 0;
}

#if IS_ENABLED(CONFIG_SPI_MASTER)

static int ad5446_write(struct ad5446_state *st, unsigned val)
{
	struct spi_device *spi = to_spi_device(st->dev);
	__be16 data = cpu_to_be16(val);

	return spi_write(spi, &data, sizeof(data));
}

static int ad5660_write(struct ad5446_state *st, unsigned val)
{
	struct spi_device *spi = to_spi_device(st->dev);
	uint8_t data[3];

	data[0] = (val >> 16) & 0xFF;
	data[1] = (val >> 8) & 0xFF;
	data[2] = val & 0xFF;

	return spi_write(spi, data, sizeof(data));
}

/**
 * ad5446_supported_spi_device_ids:
 * The AD5620/40/60 parts are available in different fixed internal reference
 * voltage options. The actual part numbers may look differently
 * (and a bit cryptic), however this style is used to make clear which
 * parts are supported here.
 */
enum ad5446_supported_spi_device_ids {
	ID_AD5300,
	ID_AD5310,
	ID_AD5320,
	ID_AD5444,
	ID_AD5446,
	ID_AD5450,
	ID_AD5451,
	ID_AD5541A,
	ID_AD5512A,
	ID_AD5553,
	ID_AD5601,
	ID_AD5611,
	ID_AD5621,
	ID_AD5641,
	ID_AD5620_2500,
	ID_AD5620_1250,
	ID_AD5640_2500,
	ID_AD5640_1250,
	ID_AD5660_2500,
	ID_AD5660_1250,
	ID_AD5662,
};

static const struct ad5446_chip_info ad5446_spi_chip_info[] = {
	[ID_AD5300] = {
		.channel = AD5446_CHANNEL_POWERDOWN(8, 16, 4),
		.write = ad5446_write,
	},
	[ID_AD5310] = {
		.channel = AD5446_CHANNEL_POWERDOWN(10, 16, 2),
		.write = ad5446_write,
	},
	[ID_AD5320] = {
		.channel = AD5446_CHANNEL_POWERDOWN(12, 16, 0),
		.write = ad5446_write,
	},
	[ID_AD5444] = {
		.channel = AD5446_CHANNEL(12, 16, 2),
		.write = ad5446_write,
	},
	[ID_AD5446] = {
		.channel = AD5446_CHANNEL(14, 16, 0),
		.write = ad5446_write,
	},
	[ID_AD5450] = {
		.channel = AD5446_CHANNEL(8, 16, 6),
		.write = ad5446_write,
	},
	[ID_AD5451] = {
		.channel = AD5446_CHANNEL(10, 16, 4),
		.write = ad5446_write,
	},
	[ID_AD5541A] = {
		.channel = AD5446_CHANNEL(16, 16, 0),
		.write = ad5446_write,
	},
	[ID_AD5512A] = {
		.channel = AD5446_CHANNEL(12, 16, 4),
		.write = ad5446_write,
	},
	[ID_AD5553] = {
		.channel = AD5446_CHANNEL(14, 16, 0),
		.write = ad5446_write,
	},
	[ID_AD5601] = {
		.channel = AD5446_CHANNEL_POWERDOWN(8, 16, 6),
		.write = ad5446_write,
	},
	[ID_AD5611] = {
		.channel = AD5446_CHANNEL_POWERDOWN(10, 16, 4),
		.write = ad5446_write,
	},
	[ID_AD5621] = {
		.channel = AD5446_CHANNEL_POWERDOWN(12, 16, 2),
		.write = ad5446_write,
	},
	[ID_AD5641] = {
		.channel = AD5446_CHANNEL_POWERDOWN(14, 16, 0),
		.write = ad5446_write,
	},
	[ID_AD5620_2500] = {
		.channel = AD5446_CHANNEL_POWERDOWN(12, 16, 2),
		.int_vref_mv = 2500,
		.write = ad5446_write,
	},
	[ID_AD5620_1250] = {
		.channel = AD5446_CHANNEL_POWERDOWN(12, 16, 2),
		.int_vref_mv = 1250,
		.write = ad5446_write,
	},
	[ID_AD5640_2500] = {
		.channel = AD5446_CHANNEL_POWERDOWN(14, 16, 0),
		.int_vref_mv = 2500,
		.write = ad5446_write,
	},
	[ID_AD5640_1250] = {
		.channel = AD5446_CHANNEL_POWERDOWN(14, 16, 0),
		.int_vref_mv = 1250,
		.write = ad5446_write,
	},
	[ID_AD5660_2500] = {
		.channel = AD5446_CHANNEL_POWERDOWN(16, 16, 0),
		.int_vref_mv = 2500,
		.write = ad5660_write,
	},
	[ID_AD5660_1250] = {
		.channel = AD5446_CHANNEL_POWERDOWN(16, 16, 0),
		.int_vref_mv = 1250,
		.write = ad5660_write,
	},
	[ID_AD5662] = {
		.channel = AD5446_CHANNEL_POWERDOWN(16, 16, 0),
		.write = ad5660_write,
	},
};

static const struct spi_device_id ad5446_spi_ids[] = {
	{"ad5300", ID_AD5300},
	{"ad5310", ID_AD5310},
	{"ad5320", ID_AD5320},
	{"ad5444", ID_AD5444},
	{"ad5446", ID_AD5446},
	{"ad5450", ID_AD5450},
	{"ad5451", ID_AD5451},
	{"ad5452", ID_AD5444}, /* ad5452 is compatible to the ad5444 */
	{"ad5453", ID_AD5446}, /* ad5453 is compatible to the ad5446 */
	{"ad5512a", ID_AD5512A},
	{"ad5541a", ID_AD5541A},
	{"ad5542a", ID_AD5541A}, /* ad5541a and ad5542a are compatible */
	{"ad5543", ID_AD5541A}, /* ad5541a and ad5543 are compatible */
	{"ad5553", ID_AD5553},
	{"ad5601", ID_AD5601},
	{"ad5611", ID_AD5611},
	{"ad5621", ID_AD5621},
	{"ad5641", ID_AD5641},
	{"ad5620-2500", ID_AD5620_2500}, /* AD5620/40/60: */
	{"ad5620-1250", ID_AD5620_1250}, /* part numbers may look differently */
	{"ad5640-2500", ID_AD5640_2500},
	{"ad5640-1250", ID_AD5640_1250},
	{"ad5660-2500", ID_AD5660_2500},
	{"ad5660-1250", ID_AD5660_1250},
	{"ad5662", ID_AD5662},
	{"dac081s101", ID_AD5300}, /* compatible Texas Instruments chips */
	{"dac101s101", ID_AD5310},
	{"dac121s101", ID_AD5320},
	{"dac7512", ID_AD5320},
	{}
};
MODULE_DEVICE_TABLE(spi, ad5446_spi_ids);

#ifdef CONFIG_OF
static const struct of_device_id ad5446_of_ids[] = {
	{ .compatible = "ti,dac7512" },
	{ }
};
MODULE_DEVICE_TABLE(of, ad5446_of_ids);
#endif

static int ad5446_spi_probe(struct spi_device *spi)
{
	const struct spi_device_id *id = spi_get_device_id(spi);

	return ad5446_probe(&spi->dev, id->name,
		&ad5446_spi_chip_info[id->driver_data]);
}

static int ad5446_spi_remove(struct spi_device *spi)
{
	return ad5446_remove(&spi->dev);
}

static struct spi_driver ad5446_spi_driver = {
	.driver = {
		.name	= "ad5446",
		.of_match_table = of_match_ptr(ad5446_of_ids),
	},
	.probe		= ad5446_spi_probe,
	.remove		= ad5446_spi_remove,
	.id_table	= ad5446_spi_ids,
};

static int __init ad5446_spi_register_driver(void)
{
	return spi_register_driver(&ad5446_spi_driver);
}

static void ad5446_spi_unregister_driver(void)
{
	spi_unregister_driver(&ad5446_spi_driver);
}

#else

static inline int ad5446_spi_register_driver(void) { return 0; }
static inline void ad5446_spi_unregister_driver(void) { }

#endif

#if IS_ENABLED(CONFIG_I2C)

static int ad5622_write(struct ad5446_state *st, unsigned val)
{
	struct i2c_client *client = to_i2c_client(st->dev);
	__be16 data = cpu_to_be16(val);

	return i2c_master_send(client, (char *)&data, sizeof(data));
}

/**
 * ad5446_supported_i2c_device_ids:
 * The AD5620/40/60 parts are available in different fixed internal reference
 * voltage options. The actual part numbers may look differently
 * (and a bit cryptic), however this style is used to make clear which
 * parts are supported here.
 */
enum ad5446_supported_i2c_device_ids {
	ID_AD5602,
	ID_AD5612,
	ID_AD5622,
};

static const struct ad5446_chip_info ad5446_i2c_chip_info[] = {
	[ID_AD5602] = {
		.channel = AD5446_CHANNEL_POWERDOWN(8, 16, 4),
		.write = ad5622_write,
	},
	[ID_AD5612] = {
		.channel = AD5446_CHANNEL_POWERDOWN(10, 16, 2),
		.write = ad5622_write,
	},
	[ID_AD5622] = {
		.channel = AD5446_CHANNEL_POWERDOWN(12, 16, 0),
		.write = ad5622_write,
	},
};

static int ad5446_i2c_probe(struct i2c_client *i2c,
			    const struct i2c_device_id *id)
{
	return ad5446_probe(&i2c->dev, id->name,
		&ad5446_i2c_chip_info[id->driver_data]);
}

static int ad5446_i2c_remove(struct i2c_client *i2c)
{
	return ad5446_remove(&i2c->dev);
}

static const struct i2c_device_id ad5446_i2c_ids[] = {
	{"ad5301", ID_AD5602},
	{"ad5311", ID_AD5612},
	{"ad5321", ID_AD5622},
	{"ad5602", ID_AD5602},
	{"ad5612", ID_AD5612},
	{"ad5622", ID_AD5622},
	{}
};
MODULE_DEVICE_TABLE(i2c, ad5446_i2c_ids);

static struct i2c_driver ad5446_i2c_driver = {
	.driver = {
		   .name = "ad5446",
	},
	.probe = ad5446_i2c_probe,
	.remove = ad5446_i2c_remove,
	.id_table = ad5446_i2c_ids,
};

static int __init ad5446_i2c_register_driver(void)
{
	return i2c_add_driver(&ad5446_i2c_driver);
}

static void __exit ad5446_i2c_unregister_driver(void)
{
	i2c_del_driver(&ad5446_i2c_driver);
}

#else

static inline int ad5446_i2c_register_driver(void) { return 0; }
static inline void ad5446_i2c_unregister_driver(void) { }

#endif

static int __init ad5446_init(void)
{
	int ret;

	ret = ad5446_spi_register_driver();
	if (ret)
		return ret;

	ret = ad5446_i2c_register_driver();
	if (ret) {
		ad5446_spi_unregister_driver();
		return ret;
	}

	return 0;
}
module_init(ad5446_init);

static void __exit ad5446_exit(void)
{
	ad5446_i2c_unregister_driver();
	ad5446_spi_unregister_driver();
}
module_exit(ad5446_exit);

MODULE_AUTHOR("Michael Hennerich <michael.hennerich@analog.com>");
MODULE_DESCRIPTION("Analog Devices AD5444/AD5446 DAC");
MODULE_LICENSE("GPL v2");
> } total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; spin_lock(&adev->mm_stats.lock); /* Increase the amount of accumulated us. */ time_us = ktime_to_us(ktime_get()); increment_us = time_us - adev->mm_stats.last_update_us; adev->mm_stats.last_update_us = time_us; adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, us_upper_bound); /* This prevents the short period of low performance when the VRAM * usage is low and the driver is in debt or doesn't have enough * accumulated us to fill VRAM quickly. * * The situation can occur in these cases: * - a lot of VRAM is freed by userspace * - the presence of a big buffer causes a lot of evictions * (solution: split buffers into smaller ones) * * If 128 MB or 1/8th of VRAM is free, start filling it now by setting * accum_us to a positive number. */ if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { s64 min_us; /* Be more aggressive on dGPUs. Try to fill a portion of free * VRAM now. */ if (!(adev->flags & AMD_IS_APU)) min_us = bytes_to_us(adev, free_vram / 4); else min_us = 0; /* Reset accum_us on APUs. */ adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); } /* This is set to 0 if the driver is in debt to disallow (optional) * buffer moves. */ *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); /* Do the same for visible VRAM if half of it is free */ if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { u64 total_vis_vram = adev->gmc.visible_vram_size; u64 used_vis_vram = amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); if (used_vis_vram < total_vis_vram) { u64 free_vis_vram = total_vis_vram - used_vis_vram; adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + increment_us, us_upper_bound); if (free_vis_vram >= total_vis_vram / 2) adev->mm_stats.accum_us_vis = max(bytes_to_us(adev, free_vis_vram / 2), adev->mm_stats.accum_us_vis); } *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); } else { *max_vis_bytes = 0; } spin_unlock(&adev->mm_stats.lock); } /* Report how many bytes have really been moved for the last command * submission. This can result in a debt that can stop buffer migrations * temporarily. */ void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, u64 num_vis_bytes) { spin_lock(&adev->mm_stats.lock); adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); spin_unlock(&adev->mm_stats.lock); } static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct amdgpu_cs_parser *p = param; struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false, .resv = bo->tbo.base.resv }; uint32_t domain; int r; if (bo->tbo.pin_count) return 0; /* Don't move this buffer if we have depleted our allowance * to move it. Don't move anything if the threshold is zero. */ if (p->bytes_moved < p->bytes_moved_threshold && (!bo->tbo.base.dma_buf || list_empty(&bo->tbo.base.dma_buf->attachments))) { if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { /* And don't move a CPU_ACCESS_REQUIRED BO to limited * visible VRAM if we've depleted our allowance to do * that. */ if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) domain = bo->preferred_domains; else domain = bo->allowed_domains; } else { domain = bo->preferred_domains; } } else { domain = bo->allowed_domains; } retry: amdgpu_bo_placement_from_domain(bo, domain); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); p->bytes_moved += ctx.bytes_moved; if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && amdgpu_bo_in_cpu_visible_vram(bo)) p->bytes_moved_vis += ctx.bytes_moved; if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { domain = bo->allowed_domains; goto retry; } return r; } static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, struct list_head *validated) { struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_bo_list_entry *lobj; int r; list_for_each_entry(lobj, validated, tv.head) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); struct mm_struct *usermm; usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); if (usermm && usermm != current->mm) return -EPERM; if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && lobj->user_invalidated && lobj->user_pages) { amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) return r; amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, lobj->user_pages); } r = amdgpu_cs_bo_validate(p, bo); if (r) return r; kvfree(lobj->user_pages); lobj->user_pages = NULL; } return 0; } static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct amdgpu_vm *vm = &fpriv->vm; struct amdgpu_bo_list_entry *e; struct list_head duplicates; struct amdgpu_bo *gds; struct amdgpu_bo *gws; struct amdgpu_bo *oa; int r; INIT_LIST_HEAD(&p->validated); /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ if (cs->in.bo_list_handle) { if (p->bo_list) return -EINVAL; r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, &p->bo_list); if (r) return r; } else if (!p->bo_list) { /* Create a empty bo_list when no handle is provided */ r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, &p->bo_list); if (r) return r; } /* One for TTM and one for the CS job */ amdgpu_bo_list_for_each_entry(e, p->bo_list) e->tv.num_shared = 2; amdgpu_bo_list_get_list(p->bo_list, &p->validated); INIT_LIST_HEAD(&duplicates); amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent) list_add(&p->uf_entry.tv.head, &p->validated); /* Get userptr backing pages. If pages are updated after registered * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do * amdgpu_ttm_backend_bind() to flush and invalidate new pages */ amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); bool userpage_invalidated = false; int i; e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); if (!e->user_pages) { DRM_ERROR("kvmalloc_array failure\n"); r = -ENOMEM; goto out_free_user_pages; } r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages); if (r) { kvfree(e->user_pages); e->user_pages = NULL; goto out_free_user_pages; } for (i = 0; i < bo->tbo.ttm->num_pages; i++) { if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { userpage_invalidated = true; break; } } e->user_invalidated = userpage_invalidated; } r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates); if (unlikely(r != 0)) { if (r != -ERESTARTSYS) DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); goto out_free_user_pages; } amdgpu_bo_list_for_each_entry(e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); e->bo_va = amdgpu_vm_bo_find(vm, bo); } /* Move fence waiting after getting reservation lock of * PD root. Then there is no need on a ctx mutex lock. */ r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entity); if (unlikely(r != 0)) { if (r != -ERESTARTSYS) DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n"); goto error_validate; } amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, &p->bytes_moved_vis_threshold); p->bytes_moved = 0; p->bytes_moved_vis = 0; r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, amdgpu_cs_bo_validate, p); if (r) { DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); goto error_validate; } r = amdgpu_cs_list_validate(p, &duplicates); if (r) goto error_validate; r = amdgpu_cs_list_validate(p, &p->validated); if (r) goto error_validate; amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, p->bytes_moved_vis); gds = p->bo_list->gds_obj; gws = p->bo_list->gws_obj; oa = p->bo_list->oa_obj; if (gds) { p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT; p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT; } if (gws) { p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT; p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT; } if (oa) { p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT; p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT; } if (!r && p->uf_entry.tv.bo) { struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo); r = amdgpu_ttm_alloc_gart(&uf->tbo); p->job->uf_addr += amdgpu_bo_gpu_offset(uf); } error_validate: if (r) ttm_eu_backoff_reservation(&p->ticket, &p->validated); out_free_user_pages: if (r) { amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); if (!e->user_pages) continue; amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); kvfree(e->user_pages); e->user_pages = NULL; } } return r; } static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct amdgpu_bo_list_entry *e; int r; list_for_each_entry(e, &p->validated, tv.head) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); struct dma_resv *resv = bo->tbo.base.resv; enum amdgpu_sync_mode sync_mode; sync_mode = amdgpu_bo_explicit_sync(bo) ? AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode, &fpriv->vm); if (r) return r; } return 0; } /** * amdgpu_cs_parser_fini() - clean parser states * @parser: parser structure holding parsing context. * @error: error number * @backoff: indicator to backoff the reservation * * If error is set then unvalidate buffer, otherwise just free memory * used by parsing context. **/ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff) { unsigned i; if (error && backoff) ttm_eu_backoff_reservation(&parser->ticket, &parser->validated); for (i = 0; i < parser->num_post_deps; i++) { drm_syncobj_put(parser->post_deps[i].syncobj); kfree(parser->post_deps[i].chain); } kfree(parser->post_deps); dma_fence_put(parser->fence); if (parser->ctx) { mutex_unlock(&parser->ctx->lock); amdgpu_ctx_put(parser->ctx); } if (parser->bo_list) amdgpu_bo_list_put(parser->bo_list); for (i = 0; i < parser->nchunks; i++) kvfree(parser->chunks[i].kdata); kvfree(parser->chunks); if (parser->job) amdgpu_job_free(parser->job); if (parser->uf_entry.tv.bo) { struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo); amdgpu_bo_unref(&uf); } } static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) { struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct amdgpu_device *adev = p->adev; struct amdgpu_vm *vm = &fpriv->vm; struct amdgpu_bo_list_entry *e; struct amdgpu_bo_va *bo_va; struct amdgpu_bo *bo; int r; /* Only for UVD/VCE VM emulation */ if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) { unsigned i, j; for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { struct drm_amdgpu_cs_chunk_ib *chunk_ib; struct amdgpu_bo_va_mapping *m; struct amdgpu_bo *aobj = NULL; struct amdgpu_cs_chunk *chunk; uint64_t offset, va_start; struct amdgpu_ib *ib; uint8_t *kptr; chunk = &p->chunks[i]; ib = &p->job->ibs[j]; chunk_ib = chunk->kdata; if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) continue; va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK; r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); if (r) { DRM_ERROR("IB va_start is invalid\n"); return r; } if ((va_start + chunk_ib->ib_bytes) > (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { DRM_ERROR("IB va_start+ib_bytes is invalid\n"); return -EINVAL; } /* the IB should be reserved at this point */ r = amdgpu_bo_kmap(aobj, (void **)&kptr); if (r) { return r; } offset = m->start * AMDGPU_GPU_PAGE_SIZE; kptr += va_start - offset; if (ring->funcs->parse_cs) { memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); amdgpu_bo_kunmap(aobj); r = amdgpu_ring_parse_cs(ring, p, p->job, ib); if (r) return r; } else { ib->ptr = (uint32_t *)kptr; r = amdgpu_ring_patch_cs_in_place(ring, p, p->job, ib); amdgpu_bo_kunmap(aobj); if (r) return r; } j++; } } if (!p->job->vm) return amdgpu_cs_sync_rings(p); r = amdgpu_vm_clear_freed(adev, vm, NULL); if (r) return r; r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); if (r) return r; r = amdgpu_sync_fence(&p->job->sync, fpriv->prt_va->last_pt_update); if (r) return r; if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { bo_va = fpriv->csa_va; BUG_ON(!bo_va); r = amdgpu_vm_bo_update(adev, bo_va, false); if (r) return r; r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update); if (r) return r; } amdgpu_bo_list_for_each_entry(e, p->bo_list) { /* ignore duplicates */ bo = ttm_to_amdgpu_bo(e->tv.bo); if (!bo) continue; bo_va = e->bo_va; if (bo_va == NULL) continue; r = amdgpu_vm_bo_update(adev, bo_va, false); if (r) return r; r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update); if (r) return r; } r = amdgpu_vm_handle_moved(adev, vm); if (r) return r; r = amdgpu_vm_update_pdes(adev, vm, false); if (r) return r; r = amdgpu_sync_fence(&p->job->sync, vm->last_update); if (r) return r; p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); if (amdgpu_vm_debug) { /* Invalidate all BOs to test for userspace bugs */ amdgpu_bo_list_for_each_entry(e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); /* ignore duplicates */ if (!bo) continue; amdgpu_vm_bo_invalidate(adev, bo, false); } } return amdgpu_cs_sync_rings(p); } static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, struct amdgpu_cs_parser *parser) { struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; struct amdgpu_vm *vm = &fpriv->vm; int r, ce_preempt = 0, de_preempt = 0; struct amdgpu_ring *ring; int i, j; for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { struct amdgpu_cs_chunk *chunk; struct amdgpu_ib *ib; struct drm_amdgpu_cs_chunk_ib *chunk_ib; struct drm_sched_entity *entity; chunk = &parser->chunks[i]; ib = &parser->job->ibs[j]; chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) continue; if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && (amdgpu_mcbp || amdgpu_sriov_vf(adev))) { if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) ce_preempt++; else de_preempt++; } /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ if (ce_preempt > 1 || de_preempt > 1) return -EINVAL; } r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type, chunk_ib->ip_instance, chunk_ib->ring, &entity); if (r) return r; if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT; if (parser->entity && parser->entity != entity) return -EINVAL; /* Return if there is no run queue associated with this entity. * Possibly because of disabled HW IP*/ if (entity->rq == NULL) return -EINVAL; parser->entity = entity; ring = to_amdgpu_ring(entity->rq->sched); r = amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0, AMDGPU_IB_POOL_DELAYED, ib); if (r) { DRM_ERROR("Failed to get ib !\n"); return r; } ib->gpu_addr = chunk_ib->va_start; ib->length_dw = chunk_ib->ib_bytes / 4; ib->flags = chunk_ib->flags; j++; } /* MM engine doesn't support user fences */ ring = to_amdgpu_ring(parser->entity->rq->sched); if (parser->job->uf_addr && ring->funcs->no_user_fence) return -EINVAL; return 0; } static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, struct amdgpu_cs_chunk *chunk) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; unsigned num_deps; int i, r; struct drm_amdgpu_cs_chunk_dep *deps; deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; num_deps = chunk->length_dw * 4 / sizeof(struct drm_amdgpu_cs_chunk_dep); for (i = 0; i < num_deps; ++i) { struct amdgpu_ctx *ctx; struct drm_sched_entity *entity; struct dma_fence *fence; ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); if (ctx == NULL) return -EINVAL; r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, deps[i].ip_instance, deps[i].ring, &entity); if (r) { amdgpu_ctx_put(ctx); return r; } fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); amdgpu_ctx_put(ctx); if (IS_ERR(fence)) return PTR_ERR(fence); else if (!fence) continue; if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { struct drm_sched_fence *s_fence; struct dma_fence *old = fence; s_fence = to_drm_sched_fence(fence); fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(old); } r = amdgpu_sync_fence(&p->job->sync, fence); dma_fence_put(fence); if (r) return r; } return 0; } static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, uint32_t handle, u64 point, u64 flags) { struct dma_fence *fence; int r; r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); if (r) { DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", handle, point, r); return r; } r = amdgpu_sync_fence(&p->job->sync, fence); dma_fence_put(fence); return r; } static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, struct amdgpu_cs_chunk *chunk) { struct drm_amdgpu_cs_chunk_sem *deps; unsigned num_deps; int i, r; deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; num_deps = chunk->length_dw * 4 / sizeof(struct drm_amdgpu_cs_chunk_sem); for (i = 0; i < num_deps; ++i) { r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle, 0, 0); if (r) return r; } return 0; } static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p, struct amdgpu_cs_chunk *chunk) { struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; unsigned num_deps; int i, r; syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; num_deps = chunk->length_dw * 4 / sizeof(struct drm_amdgpu_cs_chunk_syncobj); for (i = 0; i < num_deps; ++i) { r = amdgpu_syncobj_lookup_and_add_to_sync(p, syncobj_deps[i].handle, syncobj_deps[i].point, syncobj_deps[i].flags); if (r) return r; } return 0; } static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, struct amdgpu_cs_chunk *chunk) { struct drm_amdgpu_cs_chunk_sem *deps; unsigned num_deps; int i; deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; num_deps = chunk->length_dw * 4 / sizeof(struct drm_amdgpu_cs_chunk_sem); if (p->post_deps) return -EINVAL; p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), GFP_KERNEL); p->num_post_deps = 0; if (!p->post_deps) return -ENOMEM; for (i = 0; i < num_deps; ++i) { p->post_deps[i].syncobj = drm_syncobj_find(p->filp, deps[i].handle); if (!p->post_deps[i].syncobj) return -EINVAL; p->post_deps[i].chain = NULL; p->post_deps[i].point = 0; p->num_post_deps++; } return 0; } static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p, struct amdgpu_cs_chunk *chunk) { struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; unsigned num_deps; int i; syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; num_deps = chunk->length_dw * 4 / sizeof(struct drm_amdgpu_cs_chunk_syncobj); if (p->post_deps) return -EINVAL; p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), GFP_KERNEL); p->num_post_deps = 0; if (!p->post_deps) return -ENOMEM; for (i = 0; i < num_deps; ++i) { struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; dep->chain = NULL; if (syncobj_deps[i].point) { dep->chain = dma_fence_chain_alloc(); if (!dep->chain) return -ENOMEM; } dep->syncobj = drm_syncobj_find(p->filp, syncobj_deps[i].handle); if (!dep->syncobj) { dma_fence_chain_free(dep->chain); return -EINVAL; } dep->point = syncobj_deps[i].point; p->num_post_deps++; } return 0; } static int amdgpu_cs_dependencies(struct amdgpu_device *adev, struct amdgpu_cs_parser *p) { int i, r; /* TODO: Investigate why we still need the context lock */ mutex_unlock(&p->ctx->lock); for (i = 0; i < p->nchunks; ++i) { struct amdgpu_cs_chunk *chunk; chunk = &p->chunks[i]; switch (chunk->chunk_id) { case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: r = amdgpu_cs_process_fence_dep(p, chunk); if (r) goto out; break; case AMDGPU_CHUNK_ID_SYNCOBJ_IN: r = amdgpu_cs_process_syncobj_in_dep(p, chunk); if (r) goto out; break; case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: r = amdgpu_cs_process_syncobj_out_dep(p, chunk); if (r) goto out; break; case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk); if (r) goto out; break; case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk); if (r) goto out; break; } } out: mutex_lock(&p->ctx->lock); return r; } static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) { int i; for (i = 0; i < p->num_post_deps; ++i) { if (p->post_deps[i].chain && p->post_deps[i].point) { drm_syncobj_add_point(p->post_deps[i].syncobj, p->post_deps[i].chain, p->fence, p->post_deps[i].point); p->post_deps[i].chain = NULL; } else { drm_syncobj_replace_fence(p->post_deps[i].syncobj, p->fence); } } } static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct drm_sched_entity *entity = p->entity; struct amdgpu_bo_list_entry *e; struct amdgpu_job *job; uint64_t seq; int r; job = p->job; p->job = NULL; r = drm_sched_job_init(&job->base, entity, &fpriv->vm); if (r) goto error_unlock; drm_sched_job_arm(&job->base); /* No memory allocation is allowed while holding the notifier lock. * The lock is held until amdgpu_cs_submit is finished and fence is * added to BOs. */ mutex_lock(&p->adev->notifier_lock); /* If userptr are invalidated after amdgpu_cs_parser_bos(), return * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. */ amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); } if (r) { r = -EAGAIN; goto error_abort; } p->fence = dma_fence_get(&job->base.s_fence->finished); seq = amdgpu_ctx_add_fence(p->ctx, entity, p->fence); amdgpu_cs_post_dependencies(p); if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && !p->ctx->preamble_presented) { job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; p->ctx->preamble_presented = true; } cs->out.handle = seq; job->uf_sequence = seq; amdgpu_job_free_resources(job); trace_amdgpu_cs_ioctl(job); amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket); drm_sched_entity_push_job(&job->base); amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); /* Make sure all BOs are remembered as writers */ amdgpu_bo_list_for_each_entry(e, p->bo_list) e->tv.num_shared = 0; ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); mutex_unlock(&p->adev->notifier_lock); return 0; error_abort: drm_sched_job_cleanup(&job->base); mutex_unlock(&p->adev->notifier_lock); error_unlock: amdgpu_job_free(job); return r; } static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser) { int i; if (!trace_amdgpu_cs_enabled()) return; for (i = 0; i < parser->job->num_ibs; i++) trace_amdgpu_cs(parser, i); } int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct amdgpu_device *adev = drm_to_adev(dev); union drm_amdgpu_cs *cs = data; struct amdgpu_cs_parser parser = {}; bool reserved_buffers = false; int r; if (amdgpu_ras_intr_triggered()) return -EHWPOISON; if (!adev->accel_working) return -EBUSY; parser.adev = adev; parser.filp = filp; r = amdgpu_cs_parser_init(&parser, data); if (r) { if (printk_ratelimit()) DRM_ERROR("Failed to initialize parser %d!\n", r); goto out; } r = amdgpu_cs_ib_fill(adev, &parser); if (r) goto out; r = amdgpu_cs_dependencies(adev, &parser); if (r) { DRM_ERROR("Failed in the dependencies handling %d!\n", r); goto out; } r = amdgpu_cs_parser_bos(&parser, data); if (r) { if (r == -ENOMEM) DRM_ERROR("Not enough memory for command submission!\n"); else if (r != -ERESTARTSYS && r != -EAGAIN) DRM_ERROR("Failed to process the buffer list %d!\n", r); goto out; } reserved_buffers = true; trace_amdgpu_cs_ibs(&parser); r = amdgpu_cs_vm_handling(&parser); if (r) goto out; r = amdgpu_cs_submit(&parser, cs); out: amdgpu_cs_parser_fini(&parser, r, reserved_buffers); return r; } /** * amdgpu_cs_wait_ioctl - wait for a command submission to finish * * @dev: drm device * @data: data from userspace * @filp: file private * * Wait for the command submission identified by handle to finish. */ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { union drm_amdgpu_wait_cs *wait = data; unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); struct drm_sched_entity *entity; struct amdgpu_ctx *ctx; struct dma_fence *fence; long r; ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); if (ctx == NULL) return -EINVAL; r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance, wait->in.ring, &entity); if (r) { amdgpu_ctx_put(ctx); return r; } fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle); if (IS_ERR(fence)) r = PTR_ERR(fence); else if (fence) { r = dma_fence_wait_timeout(fence, true, timeout); if (r > 0 && fence->error) r = fence->error; dma_fence_put(fence); } else r = 1; amdgpu_ctx_put(ctx); if (r < 0) return r; memset(wait, 0, sizeof(*wait)); wait->out.status = (r == 0); return 0; } /** * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence * * @adev: amdgpu device * @filp: file private * @user: drm_amdgpu_fence copied from user space */ static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, struct drm_file *filp, struct drm_amdgpu_fence *user) { struct drm_sched_entity *entity; struct amdgpu_ctx *ctx; struct dma_fence *fence; int r; ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); if (ctx == NULL) return ERR_PTR(-EINVAL); r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance, user->ring, &entity); if (r) { amdgpu_ctx_put(ctx); return ERR_PTR(r); } fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no); amdgpu_ctx_put(ctx); return fence; } int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct amdgpu_device *adev = drm_to_adev(dev); union drm_amdgpu_fence_to_handle *info = data; struct dma_fence *fence; struct drm_syncobj *syncobj; struct sync_file *sync_file; int fd, r; fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); if (IS_ERR(fence)) return PTR_ERR(fence); if (!fence) fence = dma_fence_get_stub(); switch (info->in.what) { case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: r = drm_syncobj_create(&syncobj, 0, fence); dma_fence_put(fence); if (r) return r; r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); drm_syncobj_put(syncobj); return r; case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: r = drm_syncobj_create(&syncobj, 0, fence); dma_fence_put(fence); if (r) return r; r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle); drm_syncobj_put(syncobj); return r; case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { dma_fence_put(fence); return fd; } sync_file = sync_file_create(fence); dma_fence_put(fence); if (!sync_file) { put_unused_fd(fd); return -ENOMEM; } fd_install(fd, sync_file->file); info->out.handle = fd; return 0; default: dma_fence_put(fence); return -EINVAL; } } /** * amdgpu_cs_wait_all_fences - wait on all fences to signal * * @adev: amdgpu device * @filp: file private * @wait: wait parameters * @fences: array of drm_amdgpu_fence */ static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, struct drm_file *filp, union drm_amdgpu_wait_fences *wait, struct drm_amdgpu_fence *fences) { uint32_t fence_count = wait->in.fence_count; unsigned int i; long r = 1; for (i = 0; i < fence_count; i++) { struct dma_fence *fence; unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); if (IS_ERR(fence)) return PTR_ERR(fence); else if (!fence) continue; r = dma_fence_wait_timeout(fence, true, timeout); dma_fence_put(fence); if (r < 0) return r; if (r == 0) break; if (fence->error) return fence->error; } memset(wait, 0, sizeof(*wait)); wait->out.status = (r > 0); return 0; } /** * amdgpu_cs_wait_any_fence - wait on any fence to signal * * @adev: amdgpu device * @filp: file private * @wait: wait parameters * @fences: array of drm_amdgpu_fence */ static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, struct drm_file *filp, union drm_amdgpu_wait_fences *wait, struct drm_amdgpu_fence *fences) { unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); uint32_t fence_count = wait->in.fence_count; uint32_t first = ~0; struct dma_fence **array; unsigned int i;